Javascript Instructions

Using the js_instructions parameter, you can get the scraped site after performing various Javascript instructions.

Javascript instructions can only be applied with JavaScript rendering enabled.

Javascript Instructions parameters

Parameter	Details
action	Choose between click, scrollTo, focus, value and submit (only for FORM elements).
selector	The CSS selector.
timeout	Perform a timeout after each javascript instruction. Measured in ms.

Javascript Instructions examples

GET https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D

⇡ Input

cURL

curl --request GET --url "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D"

NodeJS

const http = require("https");

const options = {
  "method": "GET",
  "hostname": "scrape.shifter.io",
  "port": null,
  "path": "/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D",
  "headers": {}
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on("data", function (chunk) {
    chunks.push(chunk);
  });

  res.on("end", function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.end();

Python

import http.client

conn = http.client.HTTPSConnection("scrape.shifter.io")

conn.request("GET", "/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D")

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))

PHP

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

Golang

package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D"

	req, _ := http.NewRequest("GET", url, nil)

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}

Java

HttpResponse<String> response = Unirest.get("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D")
  .asString();

.NET

var client = new RestClient("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D");
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);

Ruby

require 'uri'
require 'net/http'
require 'openssl'

url = URI("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com&js_instructions=%5B%7B%22action%22%3A%22scrollTo%22%2C%22selector%22%3A%22div.navFooterBackToTop%22%2C%22timeout%22%3A%205000%2C%20%22block%22%3A%20%22start%22%7D%5D")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Get.new(url)

response = http.request(request)
puts response.read_body

⇣ Output

<!doctype html>
<html>
<head>
  <title>Example Domain</title>

  <meta charset="utf-8" />
  <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <style type="text/css">
  body {
    background-color: #f0f0f2;
    margin: 0;
    padding: 0;
    font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
     
  }
  div {
    width: 600px;
    margin: 5em auto;
    padding: 2em;
    background-color: #fdfdff;
    border-radius: 0.5em;
    box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
  }
  a:link, a:visited {
    color: #38488f;
    text-decoration: none;
  }
  @media (max-width: 700px) {
    div {
      margin: 0 auto;
      width: auto;
    }
  }
  </style>   
</head>

<body>
<div>
  <h1>Example Domain</h1>
  <p>This domain is for use in illustrative examples in documents. You may use this
  domain in literature without prior coordination or asking for permission.</p>
  <p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>