Shifter Documentation
  • Getting Started
  • PROXIES
    • Rotating Residential Proxies
    • Static Residential Proxies
  • SCRAPING API
    • Web Scraping API
      • API Request
      • Basic Request
      • Rendering JavaScript
      • Custom Headers
      • Custom Cookies
      • Proxies
      • Geolocation
      • Sessions
      • Forcing Timeouts
      • Binary Files
      • Waiting for CSS
      • Screenshot
      • Extraction Rules
      • JSON Auto Parser
      • Javascript Instructions
      • Disable Stealth
      • POST Requests
      • PUT Requests
      • Conclusion
      • API Errors
  • Google SERP API
    • Search APIs
      • Google Search API
      • Google Product API
      • Google Scholar Cite API
      • Google Scholar Author API
      • Google Scholar Profiles API
      • Google Scholar API
      • Google Autocomplete API
      • Google Maps Photos API
      • Google Maps Reviews API
      • Google Reverse Image API
      • Google Events API
      • Google Finance API
      • Google Play API
      • Google Jobs API
      • Google Local Services API
      • Google Jobs Listing API
      • Google Maps API
      • Google Trends API
        • Google Trends Categories List
        • Geo Parameter Options List
    • API Response
  • Bing SERP API
    • Search API
    • API Parameters
    • Basic API Requests
  • Yandex SERP API
    • Search API
    • Basic API Requests
    • API Parameters
  • Amazon API
    • Getting Started
      • API Parameters
      • Access the API
      • Supported Domains
    • Amazon Search
    • Amazon Seller Products
    • Amazon Seller Profile
    • Amazon Seller Feedback
    • Amazon Product
    • Amazon Category
    • Amazon Bestsellers
    • Amazon Deals
  • Useful Links
    • Github
    • Shifter Homepage
    • Knowledge Base
Powered by GitBook
On this page
  • Custom Response Headers
  • Basic Request examples

Was this helpful?

  1. SCRAPING API
  2. Web Scraping API

Basic Request

If your scraping request was successful, the API will respond with the raw HTML data of your target web page URL.

If you have enabled HTTP headers, your API response will also contain the HTTP headers sent along with your original API request.

Custom Response Headers

If you inspect your API response headers, you will notice some custom headers that will give you more details about the request you made.

Header
Description

WSA-Client-Cookies

Javascript-generated cookies after the target website is loaded

WSA-Scraped-Headers

The response headers of the scraped URL

WSA-Scraped-Resolved-Url

The final scraped URL (in case of redirects)

WSA-Call-Credit-Usage

The credit usage for a request

WSA-Current-Credit-Usage

Your total credit usage after you make the request

Basic Request examples

The API request below is the simplest invocation that you can make by specifying the URL only.

GET https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com

⇡ Input

curl --request GET --url "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com"
const http = require("https");

const options = {
  "method": "GET",
  "hostname": "scrape.shifter.io"
  "port": null,
  "path": "/v1?api_key=api_key&url=https://example.com",
  "headers": {}
};

const req = http.request(options, function (res) {
  const chunks = [];

  res.on("data", function (chunk) {
    chunks.push(chunk);
  });

  res.on("end", function () {
    const body = Buffer.concat(chunks);
    console.log(body.toString());
  });
});

req.end();
import http.client

conn = http.client.HTTPSConnection("scrape.shifter.io")

conn.request("GET", "/v1?api_key=api_key&url=https://example.com")

res = conn.getresponse()
data = res.read()

print(data.decode("utf-8"))
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func main() {

	url := "https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com"

	req, _ := http.NewRequest("GET", url, nil)

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))
HttpResponse<String> response = Unirest.get("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com")
var client = new RestClient("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com")
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
require 'uri'
require 'net/http'
require 'openssl'

url = URI("https://scrape.shifter.io/v1?api_key=api_key&url=https://example.com")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
http.verify_mode = OpenSSL::SSL::VERIFY_NONE

request = Net::HTTP::Get.new(url)

response = http.request(request)
puts response.read_body

⇣ Output

<!doctype html>
<html>
<head>
  <title>Example Domain</title>

  <meta charset="utf-8" />
  <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <style type="text/css">
  body {
    background-color: #f0f0f2;
    margin: 0;
    padding: 0;
    font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
     
  }
  div {
    width: 600px;
    margin: 5em auto;
    padding: 2em;
    background-color: #fdfdff;
    border-radius: 0.5em;
    box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
  }
  a:link, a:visited {
    color: #38488f;
    text-decoration: none;
  }
  @media (max-width: 700px) {
    div {
      margin: 0 auto;
      width: auto;
    }
  }
  </style>   
</head>

<body>
<div>
  <h1>Example Domain</h1>
  <p>This domain is for use in illustrative examples in documents. You may use this
  domain in literature without prior coordination or asking for permission.</p>
  <p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
PreviousAPI RequestNextRendering JavaScript

Last updated 1 year ago

Was this helpful?