I wrote a simple node.js app and puppeteer library.
the script will simply visit a website, then scrape some element innertext and classes. and then create a .txt file and save the element inside it and repeat.
I tested it locally and it work fine, so I deployed the app to Heroku.
locally on my pc, the .txt file is created inside my my working directory, the same place with the other files.
pls how can I view the content of this text file on Heroku, to actually know if puppeteer is saving the scraped element in to it or not.
I have tried "heroku bash -a myappname", and the ls to view the content, it only display the files and folder that I pushed to Heroku.
I also tried to clone the repo using "heroku git:clone -a myapp", it only cloned and save my code files that I pushed to Heroku, I couldn’t find the created txt file, or does it mean the file is not created ?? or I just don’t know where to find it.
look at the screenshot taking from the terminal below the expected text file is not there only the content I uploaded are there.
Here is the code pushed to Heroku, pls help me make changes if there is anything wrong with it.
var http = require("http");
const fs = require("fs");
const puppeteer = require("puppeteer");
async function scrapeData(url){
var gotData = false;
const browser = await puppeteer.launch({ args: ['--no-sandbox'] });
const page = await browser.newPage();
await page.goto(url);
//wait for all 6 balls and its innertext to appear
await page.waitForFunction(() => {
return document.querySelectorAll(".ball")[2].innerText !== '' && document.querySelectorAll(".ball")[3].innerText !== '' && document.querySelectorAll(".ball")[4].innerText !== '' && document.querySelectorAll(".ball")[5].innerText !== '' && document.querySelectorAll(".ball")[6].innerText !== '' && document.querySelectorAll(".ball")[7].innerText !== '';
}, { timeout: 100000 });
//get the color and no of the six balls starting from the left
const [ball1, ball2, ball3, ball4, ball5, ball6] = await page.evaluate(() => {
const ball1 = document.querySelectorAll(".ball")[2]; const ball2 = document.querySelectorAll(".ball")[3];
const ball3 = document.querySelectorAll(".ball")[4]; const ball4 = document.querySelectorAll(".ball")[5];
const ball5 = document.querySelectorAll(".ball")[6]; const ball6 = document.querySelectorAll(".ball")[7];
//return the color and their numbers
return [
{number: ball1.innerText,color: ball1.classList[1].split('-')[1]},{number: ball2.innerText,color: ball2.classList[1].split('-')[1]},
{number: ball3.innerText,color: ball3.classList[1].split('-')[1]},{number: ball4.innerText,color: ball4.classList[1].split('-')[1]},
{number: ball5.innerText,color: ball5.classList[1].split('-')[1]},{number: ball6.innerText,color: ball6.classList[1].split('-')[1]}
];
});
//convert time to nigeria timezone
function convertTZ(date, tzString) {
return new Date((typeof date === "string" ? new Date(date) : date).toLocaleString("en-US", {timeZone: tzString}));
}
const convertDate = convertTZ(new Date(),"Africa/Lagos");
//save the returned data in to a variable
const result = convertDate.getHours()+":"+convertDate.getMinutes()+":"+convertDate.getSeconds()+" "+[`${ball1.color}-${ball1.number}`, `${ball2.color}-${ball2.number}`, `${ball3.color}-${ball3.number}`, `${ball4.color}-${ball4.number}`, `${ball5.color}-${ball5.number}`, `${ball6.color}-${ball6.number}`];
//close the browser
await browser.close();
//create a file, and name it using today date & month in nigeria timezone
var fileName = "day-"+convertDate.getDate()+"__"+"month-"+convertDate.getMonth()+".txt";
//write the result to a file
fs.appendFile(fileName,result+"\n", (err)=>{
if(err){console.log(err);}else{
gotData = true;
if(gotData == true){
//reset gotData back to false
gotData = false;
//re-run scrapeData again
callScrapeDataAgain();
}
}});
}
scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
function callScrapeDataAgain(){
scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
}
///*config.httpPort*/
//"Node server listening on port %d in %s mode", this.address().port, app.settings.env;
http.createServer((request, response)=>{
if(request.url == "/" && request.method == "GET"){
response.writeHead(200, { 'Content-Type': 'text/html' });
response.end("your app is working", "utf-8");
}
}).listen(process.env.PORT || 3000,()=>{console.log("server working")});
See Why are my file uploads missing/deleted? in the Heroku help.
Heroku doesn’t give you a real disk and any file writes are not guaranteed to persist.
They recommend you use external storage, such as Amazon S3, instead. They provide a guide to using S3 with Node.js on Heroku.