diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | src/check.js | 4 | ||||
| -rw-r--r-- | src/scraper/package-lock.json | 247 | ||||
| -rw-r--r-- | src/scraper/package.json | 1 | ||||
| -rw-r--r-- | src/scraper/scrape.js | 14 | ||||
| -rw-r--r-- | src/web/css/site.css | 0 | ||||
| -rw-r--r-- | src/web/pages/index.html | 2 |
8 files changed, 269 insertions, 4 deletions
@@ -1 +1,2 @@ -node_modules
\ No newline at end of file +node_modules +.DS_Store
\ No newline at end of file @@ -2,6 +2,8 @@ `npm i puppeteer` +`npm i mongodb` + `node [filename]` ## Credits diff --git a/src/check.js b/src/check.js new file mode 100644 index 0000000..73c58dd --- /dev/null +++ b/src/check.js @@ -0,0 +1,4 @@ +const mongodb = require('mongodb'); + +// This is for passing the parameters of the search to check and see if it already exists in the database +// if it does exist, we're gonna call another function in another file.
\ No newline at end of file diff --git a/src/scraper/package-lock.json b/src/scraper/package-lock.json index 2ea1286..1e6439e 100644 --- a/src/scraper/package-lock.json +++ b/src/scraper/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "mongodb": "^3.6.6", "puppeteer": "^8.0.0" } }, @@ -77,6 +78,14 @@ "concat-map": "0.0.1" } }, + "node_modules/bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -118,6 +127,11 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "node_modules/debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -134,6 +148,14 @@ } } }, + "node_modules/denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/devtools-protocol": { "version": "0.0.854822", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.854822.tgz", @@ -275,6 +297,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -286,6 +313,12 @@ "node": ">=8" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "node_modules/minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -302,6 +335,81 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "node_modules/mongodb": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz", + "integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==", + "dependencies": { + "bl": "^2.2.1", + "bson": "^1.1.4", + "denque": "^1.4.1", + "optional-require": "^1.0.2", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + }, + "engines": { + "node": ">=4" + }, + "optionalDependencies": { + "saslprep": "^1.0.0" + }, + "peerDependenciesMeta": { + "aws4": { + "optional": true + }, + "bson-ext": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "mongodb-extjson": { + "optional": true + }, + "snappy": { + "optional": true + } + } + }, + "node_modules/mongodb/node_modules/bl": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", + "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/mongodb/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/mongodb/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/mongodb/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -323,6 +431,14 @@ "wrappy": "1" } }, + "node_modules/optional-require": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz", + "integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -388,6 +504,11 @@ "node": ">=8" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "node_modules/progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -479,6 +600,27 @@ } ] }, + "node_modules/saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "dependencies": { + "sparse-bitfield": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -620,6 +762,11 @@ "concat-map": "0.0.1" } }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -644,6 +791,11 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -652,6 +804,11 @@ "ms": "2.1.2" } }, + "denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + }, "devtools-protocol": { "version": "0.0.854822", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.854822.tgz", @@ -752,6 +909,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -760,6 +922,12 @@ "p-locate": "^4.1.0" } }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -773,6 +941,57 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "mongodb": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.6.tgz", + "integrity": "sha512-WlirMiuV1UPbej5JeCMqE93JRfZ/ZzqE7nJTwP85XzjAF4rRSeq2bGCb1cjfoHLOF06+HxADaPGqT0g3SbVT1w==", + "requires": { + "bl": "^2.2.1", + "bson": "^1.1.4", + "denque": "^1.4.1", + "optional-require": "^1.0.2", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + }, + "dependencies": { + "bl": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", + "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -791,6 +1010,11 @@ "wrappy": "1" } }, + "optional-require": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.2.tgz", + "integrity": "sha512-HZubVd6IfHsbnpdNF/ICaSAzBUEW1TievpkjY3tB4Jnk8L7+pJ3conPzUt3Mn/6OZx9uzTDOHYPGA8/AxYHBOg==" + }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -835,6 +1059,11 @@ "find-up": "^4.0.0" } }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -896,6 +1125,24 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/src/scraper/package.json b/src/scraper/package.json index c30aba3..7269165 100644 --- a/src/scraper/package.json +++ b/src/scraper/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "mongodb": "^3.6.6", "puppeteer": "^8.0.0" } } diff --git a/src/scraper/scrape.js b/src/scraper/scrape.js index 2002130..131ae40 100644 --- a/src/scraper/scrape.js +++ b/src/scraper/scrape.js @@ -1,8 +1,14 @@ const puppeteer = require('puppeteer'); +const mongodb = require('mongodb') -async function performSearch(query) { +// Specifically for scraping +// Stores in database once finished scraping +// query is the search string (title of movie or show) and the type (music, show, movie) +// will be used + +async function performSearch(query, type) { const browser = await puppeteer.launch({ - headless: false + headless: true }); const pages = await browser.pages(); const page = pages[0]; @@ -20,6 +26,8 @@ async function performSearch(query) { } console.log(streamingPlatforms); console.log(streamingPrices); + // do the database storage + await browser.close(); } -// performSearch("sorry to bother you")
\ No newline at end of file +// performSearch('spongebob', 'tv');
\ No newline at end of file diff --git a/src/web/css/site.css b/src/web/css/site.css new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/web/css/site.css diff --git a/src/web/pages/index.html b/src/web/pages/index.html new file mode 100644 index 0000000..0f77228 --- /dev/null +++ b/src/web/pages/index.html @@ -0,0 +1,2 @@ +<!DOCTYPE html> +</html>
\ No newline at end of file |