diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/check.js | 58 | ||||
| -rw-r--r-- | src/package-lock.json (renamed from src/scraper/package-lock.json) | 0 | ||||
| -rw-r--r-- | src/package.json (renamed from src/scraper/package.json) | 0 | ||||
| -rw-r--r-- | src/scraper/scrape.js | 73 | ||||
| -rw-r--r-- | src/test.js | 5 |
5 files changed, 108 insertions, 28 deletions
diff --git a/src/check.js b/src/check.js index 73c58dd..0b52ece 100644 --- a/src/check.js +++ b/src/check.js @@ -1,4 +1,58 @@ -const mongodb = require('mongodb'); +const MongoClient = require('mongodb').MongoClient; + // This is for passing the parameters of the search to check and see if it already exists in the database -// if it does exist, we're gonna call another function in another file.
\ No newline at end of file +// if it does exist, we're gonna call another function in another file. + +const scraper = require('./scraper/scrape') + +exports.performCheck = async function performCheck(id, query, type) { + const uri = "mongodb+srv://user0:8HL0NBINt6B8mIYF@cluster0.kfyrm.mongodb.net/StreamFinder?retryWrites=true&w=majority"; + const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true }); + + var result; + await client.connect(); + const database = client.db("db"); + const tv = database.collection("tv"); + const movie = database.collection('movie') + const search = { id: `${id}` }; + // check to see if the title is already in the database + switch (type) { + case 'tv': + result = await tv.findOne(search); + break; + case 'movie': + result = await movie.findOne(search); + break; + } + if (result == null) { + await scraper.performSearch(id, query, type) + } + // (if it needed to be scraped, it now is, and its stored. next, we perform the database search for the newly saved entry) + var array = await performDatabaseSearch(id, type); // returns values + await client.close(); + return array; +} + +async function performDatabaseSearch(id, type) { + // do database search + const uri = "mongodb+srv://user0:8HL0NBINt6B8mIYF@cluster0.kfyrm.mongodb.net/StreamFinder?retryWrites=true&w=majority"; + const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true }); + var result; + await client.connect(); + const database = client.db("db"); + const tv = database.collection("tv"); + const movie = database.collection('movie') + const search = { id: `${id}` }; + // check to see if the title is already in the database + switch (type) { + case 'tv': + result = await tv.findOne(search) + break; + case 'movie': + result = await movie.findOne(search) + break; + } + client.close() + return [result.service, result.price] +}
\ No newline at end of file diff --git a/src/scraper/package-lock.json b/src/package-lock.json index 1e6439e..1e6439e 100644 --- a/src/scraper/package-lock.json +++ b/src/package-lock.json diff --git a/src/scraper/package.json b/src/package.json index 7269165..7269165 100644 --- a/src/scraper/package.json +++ b/src/package.json diff --git a/src/scraper/scrape.js b/src/scraper/scrape.js index 131ae40..605c234 100644 --- a/src/scraper/scrape.js +++ b/src/scraper/scrape.js @@ -1,33 +1,54 @@ const puppeteer = require('puppeteer'); -const mongodb = require('mongodb') +const MongoClient = require('mongodb').MongoClient; // Specifically for scraping // Stores in database once finished scraping -// query is the search string (title of movie or show) and the type (music, show, movie) -// will be used +// query is the movie or show name +// type is the media type ("tv" for tv show or "movie") -async function performSearch(query, type) { - const browser = await puppeteer.launch({ - headless: true - }); - const pages = await browser.pages(); - const page = pages[0]; - await page.goto('https://google.com'); - const searchBox = await page.$x("/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input"); - await searchBox[0].type(`${query} streaming`); - await page.keyboard.press('Enter'); - await page.waitForNavigation(); - const resultsContainer = await page.$$('.r0VsPb') - var streamingPlatforms; - var streamingPrices; - for (let i = 0; i < resultsContainer.length; i++) { - streamingPlatforms = await resultsContainer[i].$$eval('.i3LlFf', nodes => nodes.map(n => n.innerText)); - streamingPrices = await resultsContainer[i].$$eval('.V8xno', nodes => nodes.map(n => n.innerText)) +exports.performSearch = async function (id, query, type) { + try { + const uri = "mongodb+srv://user0:8HL0NBINt6B8mIYF@cluster0.kfyrm.mongodb.net/StreamFinder?retryWrites=true&w=majority"; + const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true }); + await client.connect(); + const database = client.db("db"); + const tv = database.collection("tv"); + const movie = database.collection('movie') + const browser = await puppeteer.launch({ + headless: true + }); + const pages = await browser.pages(); + const page = pages[0]; + await page.goto('https://google.com'); + const searchBox = await page.$x("/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input"); + await searchBox[0].type(`${query} ${type} streaming`); + await page.keyboard.press('Enter'); + await page.waitForNavigation(); + const resultsContainer = await page.$$('.r0VsPb') + var streamingPlatforms; + var streamingPrices; + for (let i = 0; i < resultsContainer.length; i++) { + streamingPlatforms = await resultsContainer[i].$$eval('.i3LlFf', nodes => nodes.map(n => n.innerText)); + streamingPrices = await resultsContainer[i].$$eval('.V8xno', nodes => nodes.map(n => n.innerText)) + } + if (type == "movie") { + await movie.insertOne({ + title: query, + id: id, + service: streamingPlatforms, + price: streamingPrices + }) + } else if (type == "tv") { + await tv.insertOne({ + title: query, + id: id, + service: streamingPlatforms, + price: streamingPrices + }) + } + await client.close() + await browser.close(); + } catch (error) { + console.log(error) } - console.log(streamingPlatforms); - console.log(streamingPrices); - // do the database storage - await browser.close(); } - -// performSearch('spongebob', 'tv');
\ No newline at end of file diff --git a/src/test.js b/src/test.js new file mode 100644 index 0000000..6c10475 --- /dev/null +++ b/src/test.js @@ -0,0 +1,5 @@ +const check = require('./check') + +check.performCheck('387', 'Spongebob Squarepants', 'tv').then(res => { + console.log(res) +})
\ No newline at end of file |