x-api-token: ********************
allowBackwardLinks
parameter.{
"url": "http://example.com",
"limit": 10000,
"excludePaths": [
"string"
],
"includePaths": [
"string"
],
"maxDepth": 10,
"maxDiscoveryDepth": 0,
"ignoreSitemap": false,
"ignoreQueryParameters": false,
"deduplicateSimilarURLs": true,
"regexOnFullURL": true,
"allowBackwardLinks": false,
"allowExternalLinks": false,
"delay": 0,
"scrapeOptions": {
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"string"
],
"excludeTags": [
"string"
],
"headers": {},
"waitFor": 0,
"timeout": 30000
},
"browserOptions": {
"sessionName": "string",
"sessionTTL": "string",
"sessionRecording": "string",
"proxyCountry": "string",
"proxyURL": "string",
"fingerprint": "string"
}
}
curl --location --request POST 'https://api.scrapeless.com/api/v2/crawler/crawl' \
--header 'Content-Type: application/json' \
--header 'x-api-token;' \
--data-raw '{
"url": "http://example.com",
"limit": 10000,
"excludePaths": [
"string"
],
"includePaths": [
"string"
],
"maxDepth": 10,
"maxDiscoveryDepth": 0,
"ignoreSitemap": false,
"ignoreQueryParameters": false,
"deduplicateSimilarURLs": true,
"regexOnFullURL": true,
"allowBackwardLinks": false,
"allowExternalLinks": false,
"delay": 0,
"scrapeOptions": {
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"string"
],
"excludeTags": [
"string"
],
"headers": {},
"waitFor": 0,
"timeout": 30000
},
"browserOptions": {
"sessionName": "string",
"sessionTTL": "string",
"sessionRecording": "string",
"proxyCountry": "string",
"proxyURL": "string",
"fingerprint": "string"
}
}'
{
"success": true,
"id": "string"
}