41 function getURLContent($url, $postorget =
'GET', $param =
'', $followlocation = 1, $addheaders = array(), $allowedschemes = array(
'http',
'https'), $localurl = 0, $ssl_verifypeer = -1)
45 $USE_PROXY = empty($conf->global->MAIN_PROXY_USE) ? 0 : $conf->global->MAIN_PROXY_USE;
46 $PROXY_HOST = empty($conf->global->MAIN_PROXY_HOST) ? 0 : $conf->global->MAIN_PROXY_HOST;
47 $PROXY_PORT = empty($conf->global->MAIN_PROXY_PORT) ? 0 : $conf->global->MAIN_PROXY_PORT;
48 $PROXY_USER = empty($conf->global->MAIN_PROXY_USER) ? 0 : $conf->global->MAIN_PROXY_USER;
49 $PROXY_PASS = empty($conf->global->MAIN_PROXY_PASS) ? 0 : $conf->global->MAIN_PROXY_PASS;
51 dol_syslog(
"getURLContent postorget=".$postorget.
" URL=".$url.
" param=".$param);
60 curl_setopt($ch, CURLOPT_VERBOSE, 1);
61 curl_setopt($ch, CURLOPT_USERAGENT,
'Dolibarr geturl function');
65 @curl_setopt($ch, CURLOPT_FOLLOWLOCATION,
false);
67 if (is_array($addheaders) && count($addheaders)) {
68 curl_setopt($ch, CURLOPT_HTTPHEADER, $addheaders);
70 curl_setopt($ch, CURLINFO_HEADER_OUT,
true);
74 if (!empty($conf->global->MAIN_CURL_SSLVERSION)) {
75 curl_setopt($ch, CURLOPT_SSLVERSION, $conf->global->MAIN_CURL_SSLVERSION);
80 if ($ssl_verifypeer < 0) {
81 global $dolibarr_main_prod;
82 $ssl_verifypeer = ($dolibarr_main_prod ? true :
false);
84 if (!empty($conf->global->MAIN_CURL_DISABLE_VERIFYPEER)) {
89 curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, ($ssl_verifypeer ?
true :
false));
90 curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, ($ssl_verifypeer ?
true :
false));
94 if (is_array($allowedschemes)) {
95 foreach ($allowedschemes as $allowedscheme) {
96 if ($allowedscheme ==
'http') {
97 $protocols |= CURLPROTO_HTTP;
99 if ($allowedscheme ==
'https') {
100 $protocols |= CURLPROTO_HTTPS;
103 curl_setopt($ch, CURLOPT_PROTOCOLS, $protocols);
104 curl_setopt($ch, CURLOPT_REDIR_PROTOCOLS, $protocols);
107 curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, empty($conf->global->MAIN_USE_CONNECT_TIMEOUT) ? 5 : $conf->global->MAIN_USE_CONNECT_TIMEOUT);
108 curl_setopt($ch, CURLOPT_TIMEOUT, empty($conf->global->MAIN_USE_RESPONSE_TIMEOUT) ? 30 : $conf->global->MAIN_USE_RESPONSE_TIMEOUT);
116 curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
117 if ($postorget ==
'POST') {
118 curl_setopt($ch, CURLOPT_POST, 1);
119 curl_setopt($ch, CURLOPT_POSTFIELDS, $param);
120 } elseif ($postorget ==
'POSTALREADYFORMATED') {
121 curl_setopt($ch, CURLOPT_CUSTOMREQUEST,
'POST');
122 curl_setopt($ch, CURLOPT_POSTFIELDS, $param);
123 } elseif ($postorget ==
'PUT') {
125 curl_setopt($ch, CURLOPT_CUSTOMREQUEST,
'PUT');
126 if (!is_array($param)) {
127 parse_str($param, $array_param);
129 dol_syslog(
"parameter param must be a string", LOG_WARNING);
130 $array_param = $param;
132 curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($array_param));
133 } elseif ($postorget ==
'PUTALREADYFORMATED') {
134 curl_setopt($ch, CURLOPT_CUSTOMREQUEST,
'PUT');
135 curl_setopt($ch, CURLOPT_POSTFIELDS, $param);
136 } elseif ($postorget ==
'HEAD') {
137 curl_setopt($ch, CURLOPT_CUSTOMREQUEST,
'HEAD');
138 curl_setopt($ch, CURLOPT_NOBODY,
true);
139 } elseif ($postorget ==
'DELETE') {
140 curl_setopt($ch, CURLOPT_CUSTOMREQUEST,
'DELETE');
142 curl_setopt($ch, CURLOPT_POST, 0);
147 dol_syslog(
"getURLContent set proxy to ".$PROXY_HOST.
":".$PROXY_PORT.
" - ".$PROXY_USER.
":".$PROXY_PASS);
149 curl_setopt($ch, CURLOPT_PROXY, $PROXY_HOST.
":".$PROXY_PORT);
151 curl_setopt($ch, CURLOPT_PROXYUSERPWD, $PROXY_USER.
":".$PROXY_PASS);
161 if ($maxRedirection < 1) {
165 curl_setopt($ch, CURLOPT_URL, $newUrl);
168 $newUrlArray = parse_url($newUrl);
169 $hosttocheck = $newUrlArray[
'host'];
170 $hosttocheck = str_replace(array(
'[',
']'),
'', $hosttocheck);
173 if (in_array($hosttocheck, array(
'metadata.google.internal'))) {
174 $info[
'http_code'] = 400;
175 $info[
'content'] =
'Error bad hostname '.$hosttocheck.
' (Used by Google metadata). This value for hostname is not allowed.';
180 if (in_array($hosttocheck, array(
'localhost',
'localhost.domain'))) {
181 $iptocheck =
'127.0.0.1';
182 } elseif (in_array($hosttocheck, array(
'ip6-localhost',
'ip6-loopback'))) {
186 if (function_exists(
'gethostbyname')) {
187 $iptocheck = gethostbyname($hosttocheck);
189 $iptocheck = $hosttocheck;
195 if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_IPV6)) {
200 if ($localurl == 0) {
201 if (!filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
203 $info[
'http_code'] = 400;
204 $info[
'content'] =
'Error bad hostname IP (private or reserved range). Must be an external URL.';
207 if (!empty($_SERVER[
"SERVER_ADDR"]) && $iptocheck == $_SERVER[
"SERVER_ADDR"]) {
208 $info[
'http_code'] = 400;
209 $info[
'content'] =
'Error bad hostname IP (IP is a local IP). Must be an external URL.';
212 if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && in_array($iptocheck, explode(
',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
213 $info[
'http_code'] = 400;
214 $info[
'content'] =
'Error bad hostname IP (IP is a local IP defined into MAIN_SECURITY_SERVER_IP). Must be an external URL.';
218 if ($localurl == 1) {
220 if (filter_var($iptocheck, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
221 $info[
'http_code'] = 400;
222 $info[
'content'] =
'Error bad hostname '.$iptocheck.
'. Must be a local URL.';
225 if (!empty($conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP) && !in_array($iptocheck, explode(
',', $conf->global->MAIN_SECURITY_ANTI_SSRF_SERVER_IP))) {
226 $info[
'http_code'] = 400;
227 $info[
'content'] =
'Error bad hostname IP (IP is not a local IP defined into list MAIN_SECURITY_SERVER_IP). Must be a local URL in allowed list.';
235 $arrayofmetadataserver = array(
236 '100.100.100.200' =>
'Alibaba',
237 '192.0.0.192' =>
'Oracle',
238 '192.80.8.124' =>
'Packet',
239 '100.88.222.5' =>
'Tencent cloud',
241 foreach ($arrayofmetadataserver as $ipofmetadataserver => $nameofmetadataserver) {
242 if ($iptocheck == $ipofmetadataserver) {
243 $info[
'http_code'] = 400;
244 $info[
'content'] =
'Error bad hostname IP (Used by '.$nameofmetadataserver.
' metadata server). This IP is forbidden.';
250 if (defined(
'CURLOPT_CONNECT_TO')) {
251 $connect_to = array(sprintf(
"%s:%d:%s:%d", $newUrlArray[
'host'], empty($newUrlArray[
'port'])?
'':$newUrlArray[
'port'], $iptocheck, empty($newUrlArray[
'port'])?
'':$newUrlArray[
'port']));
254 curl_setopt($ch, CURLOPT_CONNECT_TO, $connect_to);
259 $response = curl_exec($ch);
261 $info = curl_getinfo($ch);
262 $http_code = $info[
'http_code'];
264 if ($followlocation && ($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307)) {
265 $newUrl = $info[
'redirect_url'];
272 }
while ($http_code);
274 $request = curl_getinfo($ch, CURLINFO_HEADER_OUT);
276 dol_syslog(
"getURLContent request=".$request);
277 if (!empty($conf->global->MAIN_GETURLCONTENT_OUTPUT_RESPONSE)) {
279 dol_syslog(
"getURLContent response =".$response);
281 dol_syslog(
"getURLContent response size=".strlen($response));
284 if (curl_errno($ch)) {
286 $rep[
'content'] = $response;
289 $rep[
'curl_error_no'] = curl_errno($ch);
290 $rep[
'curl_error_msg'] = curl_error($ch);
292 dol_syslog(
"getURLContent response array is ".join(
',', $rep));
300 dol_syslog(
"getURLContent http_code=".$rep[
'http_code']);
304 $rep[
'content'] = $response;
306 $rep[
'curl_error_no'] =
'';
307 $rep[
'curl_error_msg'] =
'';
327 $tmpdomain = preg_replace(
'/^https?:\/\//i',
'', $url);
328 $tmpdomain = preg_replace(
'/\/.*$/i',
'', $tmpdomain);
330 $tmpdomain = preg_replace(
'/^.*\.([^\.]+)\.([^\.]+)\.([^\.]+)$/',
'\1.\2.\3', $tmpdomain);
332 $tmpdomain = preg_replace(
'/^.*\.([^\.]+)\.([^\.]+)$/',
'\1.\2', $tmpdomain);
335 $tmpdomain = preg_replace(
'/\.[^\.]+$/',
'', $tmpdomain);
354 if (preg_match(
'/^(https?:\/\/)/i', $tmpurl, $reg)) {
357 $tmpurl = preg_replace(
'/^https?:\/\//i',
'', $tmpurl);
358 $tmpurl = preg_replace(
'/\/.*$/i',
'', $tmpurl);
360 return $prefix.$tmpurl;
371 $content = preg_replace(
'/<!--[^\-]+-->/',
'', $content);
dol_syslog($message, $level=LOG_INFO, $ident=0, $suffixinfilename='', $restricttologhandler='', $logcontext=null)
Write log message into outputs.
getDomainFromURL($url, $mode=0)
Function get second level domain name.
getRootURLFromURL($url)
Function root url from a long url For example: https://www.abc.mydomain.com/dir/page....
getURLContent($url, $postorget='GET', $param='', $followlocation=1, $addheaders=array(), $allowedschemes=array('http', 'https'), $localurl=0, $ssl_verifypeer=-1)
Function to get a content from an URL (use proxy if proxy defined).
removeHtmlComment($content)
Function to remove comments into HTML content.