Waiting for CSS

Using the wait_for_css parameter, you can make the API wait for 10 seconds until a CSS selector appears. The waiting time is set using the timeout parameter; in case of failure, its value can be customized.

If after 10 seconds, the CSS selector does not appear in DOM, the request is marked as failed, and thus, you will not be charged for it.

You can find on the right side a sample selector for the wait_for_css parameter. The API will look for an element that has .firstHeading class.

Wait for CSS examples

GET https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1

⇡ Input

curl --request GET --url "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1"

const http = require("https");

const options = {
  "method": "GET",
  "hostname": "scrape.shifter.io",
  "port": null,
  "path": "/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1",
  "headers": {}
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on("data", function (chunk) {
    chunks.push(chunk);
  });

  res.on("end", function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.end();

import http.client

conn = http.client.HTTPSConnection("scrape.shifter.io")

conn.request("GET", "/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1")

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1"

	req, _ := http.NewRequest("GET", url, nil)

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1")
  .asString();

var client = new RestClient("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);

require 'uri'
require 'net/http'
require 'openssl'

url = URI("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Get.new(url)

response = http.request(request)
puts response.read_body

⇣ Output

<!doctype html>
<html>
<head>
  <title>Example Domain</title>

  <meta charset="utf-8" />
  <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <style type="text/css">
  body {
    background-color: #f0f0f2;
    margin: 0;
    padding: 0;
    font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
     
  }
  div {
    width: 600px;
    margin: 5em auto;
    padding: 2em;
    background-color: #fdfdff;
    border-radius: 0.5em;
    box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
  }
  a:link, a:visited {
    color: #38488f;
    text-decoration: none;
  }
  @media (max-width: 700px) {
    div {
      margin: 0 auto;
      width: auto;
    }
  }
  </style>   
</head>

<body>
<div>
  <h1>Example Domain</h1>
  <p>This domain is for use in illustrative examples in documents. You may use this
  domain in literature without prior coordination or asking for permission.</p>
  <p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>

PreviousBinary Files NextScreenshot

Last updated 1 year ago

Was this helpful?

const http = require("https"); const options = { "method": "GET", "hostname": "scrape.shifter.io", "port": null, "path": "/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1", "headers": {} }; const req = http.request(options, function (res) { const chunks = []; res.on("data", function (chunk) { chunks.push(chunk); }); res.on("end", function () { const body = Buffer.concat(chunks); console.log(body.toString()); }); }); req.end();

import http.client conn = http.client.HTTPSConnection("scrape.shifter.io") conn.request("GET", "/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1") res = conn.getresponse() data = res.read() print(data.decode("utf-8"))

<?php $curl = curl_init(); curl_setopt_array($curl, [ CURLOPT_URL => "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1", CURLOPT_RETURNTRANSFER => true, CURLOPT_ENCODING => "", CURLOPT_MAXREDIRS => 10, CURLOPT_TIMEOUT => 30, CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, CURLOPT_CUSTOMREQUEST => "GET", ]); $response = curl_exec($curl); $err = curl_error($curl); curl_close($curl); if ($err) { echo "cURL Error #:" . $err; } else { echo $response; }

package main import ( "fmt" "net/http" "io/ioutil" ) func main() { url := "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1" req, _ := http.NewRequest("GET", url, nil) res, _ := http.DefaultClient.Do(req) defer res.Body.Close() body, _ := ioutil.ReadAll(res.Body) fmt.Println(res) fmt.Println(string(body)) }

var client = new RestClient("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1"); var request = new RestRequest(Method.GET); IRestResponse response = client.Execute(request);

require 'uri' require 'net/http' require 'openssl' url = URI("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&wait_for_css=.firstHeading&render_js=1") http = Net::HTTP.new(url.host, url.port) http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_NONE request = Net::HTTP::Get.new(url) response = http.request(request) puts response.read_body

<!doctype html> <html> <head> <title>Example Domain</title> <meta charset="utf-8" /> <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <style type="text/css"> body { background-color: #f0f0f2; margin: 0; padding: 0; font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; } div { width: 600px; margin: 5em auto; padding: 2em; background-color: #fdfdff; border-radius: 0.5em; box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02); } a:link, a:visited { color: #38488f; text-decoration: none; } @media (max-width: 700px) { div { margin: 0 auto; width: auto; } } </style> </head> <body> <div> <h1>Example Domain</h1> <p>This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.</p> <p><a href="https://www.iana.org/domains/example">More information...</a></p> </div> </body> </html>