I’ve made a node.js web scraper code that runs fine on my computer, however, when I deploy to my Google Cloud VM instance running Debian, it returns a timeout error for a specific website. I’ve tried many differnent setups for puppeteer, but none seems to work. I believe the website I’m trying to scrape is blocking my code when I run from the google cloud server, but not when I run from my computer. The scraping part is working fine on my computer. Puppeteer finds the HTML tags and retrieve the info.
const puppeteer = require('puppeteer');
const GoogleSpreadsheet = require('google-spreadsheet');
const { promisify } = require('util');
const credentials = require('./credentials.json');
async function main(){
const scrapCopasa = await scrapCopasaFunction();
console.log('Done!')
}
async function scrapCopasaFunction() {
const browser = await puppeteer.launch({
args: ['--no-sandbox'],
});
const page = await browser.newPage();
//await page.setDefaultNavigationTimeout(0);
//await page.setViewport({width: 1366, height: 768});
await page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36');
await page.goto('http://www.copasa.com.br/wps/portal/internet/abastecimento-de-agua/nivel-dos-reservatorios');
//await new Promise(resolve => setTimeout(resolve, 5000));
let isUsernameNotFound = await page.evaluate(() => {
if(document.getElementsByClassName('h2')[0]) {
if(document.getElementsByTagName('h2')[0].textContent == "Sorry, this page isn't available.") {
return true;
}
}
});
if(isUsernameNotFound) {
console.log('Account not exists!');
await browser.close();
return;
}
let reservoirLevelsCopasa = await page.evaluate(() => {
const tds = Array.from(document.querySelectorAll('table tr td'))
return tds.map(td => td.innerText)
});
const riomanso = reservoirLevelsCopasa[13].replace(",",".").substring(0,5);
const serraazul = reservoirLevelsCopasa[17].replace(",",".").substring(0,5);
const vargemdasflores = reservoirLevelsCopasa[21].replace(",",".").substring(0,5);
await browser.close();
return[riomanso, serraazul, vargemdasflores];
}
main();
And error that I’m getting is the following:
(node:6425) UnhandledPromiseRejectionWarning: TimeoutError: Navigation Timeout Exceeded: 30000ms exceeded
at /home/xxx/reservoirs/node_modules/puppeteer/lib/LifecycleWatcher.js:142:21
at async FrameManager.navigateFrame (/home/xxx/reservoirs/node_modules/puppeteer/lib/FrameManager.js:94:17)
at async Frame.goto (/home/xxx/reservoirs/node_modules/puppeteer/lib/FrameManager.js:406:12)
at async Page.goto (/home/xxx/reservoirs/node_modules/puppeteer/lib/Page.js:674:12)
at async scrapCopasaFunction (/home/xxx/reservoirs/reservatorios.js:129:5)
at async main (/home/xxx/reservoirs/reservatorios.js:9:25)
-- ASYNC --
at Frame.<anonymous> (/home/xxx/reservoirs/node_modules/puppeteer/lib/helper.js:111:15)
at Page.goto (/home/xxx/reservoirs/node_modules/puppeteer/lib/Page.js:674:49)
at Page.<anonymous> (/home/xxx/reservoirs/node_modules/puppeteer/lib/helper.js:112:23)
at scrapCopasaFunction (/home/xxx/reservoirs/reservatorios.js:129:16)
at processTicksAndRejections (internal/process/task_queues.js:93:5)
at async main (/home/xxx/reservoirs/reservatorios.js:9:25)
(Use `node --trace-warnings ...` to show where the warning was created)
(node:6425) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async f
unction without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled
promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode)
. (rejection id: 1)
(node:6425) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not ha
ndled will terminate the Node.js process with a non-zero exit code.
a modafinil prescription – modafinil and birth control modafinil 200 mg
free datingwebsites
herpes dating sites
20 mg cialis best price – generic tadalafil 5mg price cialis 20 mg cost
chubby gay dating
japan gay dating site
13 year old gay dating sites
stromectol for humans for sale – ivermectin 0.5 lotion where to buy stromectol
real money online casino – sugarhouse casino online slot machines
gay chub/chaser dating
dating sites for hpv gay
[url=”http://freegaychatnew.com?”]teenage gay dating[/url]
what is the latest treatment for erectile dysfunction – best ed pills what is the most effective pill for ed
prednisone pill 20 mg – prednisone 5mg for sale buy prednisone 5mg
sildenafil 100mg price australia – sildenafil tablets online in india how to get cheap viagra online
dating a older gay men points
rust gay dating sites
gay dating sight
brand cialis buy – Cialis for sale buy cialis canadian pharmacy
gay cock mature dating
gay dating sites for men bdsm
gay black dating
ivermectin 6mg tablet for lice – ivermectin cost in usa stromectol ivermectin 3 mg
dr oz erectile dysfunction pills – medication for ed blue pill for ed
ventolin 90 – gnrventolin buy Albuterol
cytotec pharmacy uk – cytotec buy online cytotec 25 mcg
doxycycline 150 mg price – order doxycycline without prescription doxycycline for dogs
neurontin 1000 mg – neurontin 300 mg price cost of synthroid medication in canada
female viagra pharmacy australia – sildenafil 75 mg viagra tablets price in uk
mypharmacy – where can i buy cheap cialis
levitra vardenafil – generic vardenafil online how does vardenafil work
gay speed dating math
gay asian dating site
free gay dating websites uk
where to get accutane prescription – accutane 20 mg price in india accutane online canada pharmacy
where to buy amoxicillin – buy amoxicilin 500 buy amoxicillin 500 mg online mexico
plenty fish gay dating
white asian gay dating
tinder for gay dating
affordable term papers – wtessay buy a custom essay
gay dating men most popular
growlr gay dating site
gay dating sites 30s
viagra 100mg uk – Buy viagra without rx sildenafil 100 mg prescription
generic stromectol for humans – ivermectinok.com purchase stromectol
ai gay dating
dating white gay guy
gay only web dating sites
prednisone 10mg cost – no prescription prednisone canadian pharmacy deltasone 20 mg tablet
furosemide 10 mg price – buy clomid online lasix 20mg tablet price
gay dating 2021 -datingnews
gay amish dating
gay leather dating
ventolin cost – buy Albuterol ventolin price
cytotec pill for sale – where can i get cytotec pills in south africa buy cytotec online australia
doxycycline 75 mg price – prednisolone sodium phosphate prednisolone 25mg
buy ivermectin for humans uk – ivermectin cream ivermectin 250ml
ivermectin tablets for sale walmart – ivermectin 0.5 ivermectin 1
sildenafil reviews – buy sildenafil online buy sildenafil
buy generic tadalafil online – online tadalafil tadalafil 10 mg
isotretinoin 5 mg – purchase accutane accutane where to buy
pay for paper writing – how to write an essay about my family write essay service
real viagra without prescription – Cialis pharmacy online cialis 5 coupon
vardenafil 40 mg – cvs pharmacy new ed treatments
ivermectin in australia ivermectin for cattle
mayo clinic hydroxychloroquine – prednisone online pharmacy prednisone cost
buy cenforce 200mg online paypal – fildena 120 cenforce 200 mg – cenforce 100 usa
xenical capsule – orlistat medication coupon generic xenical australia
india ivermectin – stromecti.com buy ivermectin
viagra brand name – viagra 200mg price cost of viagra prescription
cialis 5 mg tablet – cialis 5mg cost cialis pills 2.5mg
buying prednisone from canada – prednisone 20 mg buy online buy prednisone no prescription
discount cialis prices – cilaspl.com buying tadalafil in mexico
essay reworder
cheap papers
cost of ivermectin 1% cream – stromectol tablet price cost of ivermectin lotion
stromectol tablets for humans – stromectol price ivermectin buy uk
type my essay for me
personal statement editing
canadian pharmacy without prescription – generic cialis how much is daily cialis
secure medical online pharmacy – Buy cialis online cheap average cost of cialis 20mg
casino online real money – wind creek casino online games online casino gambling
ivermectin uk – buy ivermectin 6mg online buy ivermectin 12mg for humans
is ivermectin a prescription drug – ivermectin 12 mg stromectol ivermectin topical
54 prednisone – prednixone tables for sale how to get prednisone prescription
sildenafil 100mg sale – cialis 40mg cheap tadalafil tablets
acheter 50mg viagra – viagra 50mg generika rezeptfrei kaufen vrai tadalafil 5mg prix
ivermectin 0.5 – stromectol 3mg oral ivermectin 5
buy prednisone 40mg sale – canadian pharmaceuticals online safe prednisone 10mg ca
order accutane 40mg generic – order lipitor 80mg generic amoxicillin drug
pregabalin 75mg pills – order furosemide sale order azithromycin 500mg online
generic cialis online fast shipping tadalafil generic where to buy
order cenforce 100mg online – cenforce 100mg us acyclovir 800mg over the counter
buy tizanidine 2mg online – toradol 10mg brand colchicine order online
tadalafil drug tadalafil price walmart
cialis at canadian pharmacy buy tadalafil
lowest price cialis https://cialisbusd.com/
cialis without a prescription tadalafil goodrx
cialis without prescription https://cialisicp.com/
https://cialiswbtc.com/ lowest price cialis
stromectol 12 mg
http://stromectolusdt.com/ buy 12mg ivermectin
canadian pharmacy stromectol generic stromectol 12mg