commit a38f378e1681d66bffa503a155e29e877317f307 Author: shenfan Date: Tue Apr 1 16:08:47 2025 +0800 代码初始化 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d04b0fc --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +data +venv +**/log +.idea +**/__pycache__/ +test.py +config.py +Pipfile +Pipfile.lock +run.bat \ No newline at end of file diff --git a/base/__init__.py b/base/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/base/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/base/base_spider.py b/base/base_spider.py new file mode 100644 index 0000000..2756d19 --- /dev/null +++ b/base/base_spider.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from abc import ABC, abstractclassmethod + + +class AbstractSpider(ABC): + """ + 爬虫抽象类 + """ + def init_config(self): + """ + 初始化配置 + :return: + """ + pass + + def start(self, task_id): + """ + 运行爬虫 + :return: + """ diff --git a/base/enums.py b/base/enums.py new file mode 100644 index 0000000..8a0c553 --- /dev/null +++ b/base/enums.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from enum import Enum + + +class TaskStatus(Enum): + WAITING = 1 + RUNNING = 2 + COMPLETED = 3 + FAIL = 4 + + def __eq__(self, other): + return self.value == other + + +class Platform(Enum): + XIN_HUA = "xin_hua" + REN_MIN = "ren_min" + YANG_SHI = "yang_shi" + + def __eq__(self, other): + return self.value == other + + +if __name__ == '__main__': + print(Platform.REN_MIN == "ren_min") diff --git a/db.py b/db.py new file mode 100644 index 0000000..a90ee73 --- /dev/null +++ b/db.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +from tortoise import Tortoise, run_async +from config import * +from utils.utils import logger + +""" +数据库操作 +""" + + +def get_db_url(): + """ + 拼接数据库url + :return: + """ + return F"mysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_DATABASE}" + + +async def init(): + """ + 初始化数据库连接 + :return: + """ + await Tortoise.init( + db_url=get_db_url(), + modules={"models": ['models.monitor_result_db', 'models.monitor_task_db']} + ) + await Tortoise.generate_schemas() + logger.info("[数据库]初始化数据库连接成功") + diff --git a/lib/stealth.min.js b/lib/stealth.min.js new file mode 100644 index 0000000..b4ca68f --- /dev/null +++ b/lib/stealth.min.js @@ -0,0 +1,7 @@ +/*! + * Note: Auto-generated, do not update manually. + * Generated by: https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions + * Generated on: Mon, 05 Jun 2023 06:17:57 GMT + * License: MIT + */ +(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:'utils => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, \'chrome\')`\n Object.defineProperty(window, \'chrome\', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We\'ll extend that later\n })\n }\n\n // That means we\'re running headful and don\'t need to mock anything\n if (\'app\' in window.chrome) {\n return // Nothing to do here\n }\n\n const makeError = {\n ErrorInInvocation: fn => {\n const err = new TypeError(`Error in invocation of app.${fn}()`)\n return utils.stripErrorWithAnchor(\n err,\n `at ${fn} (eval at `\n )\n }\n }\n\n // There\'s a some static data in that property which doesn\'t seem to change,\n // we should periodically check for updates: `JSON.stringify(window.app, null, 2)`\n const STATIC_DATA = JSON.parse(\n `\n{\n "isInstalled": false,\n "InstallState": {\n "DISABLED": "disabled",\n "INSTALLED": "installed",\n "NOT_INSTALLED": "not_installed"\n },\n "RunningState": {\n "CANNOT_RUN": "cannot_run",\n "READY_TO_RUN": "ready_to_run",\n "RUNNING": "running"\n }\n}\n `.trim()\n )\n\n window.chrome.app = {\n ...STATIC_DATA,\n\n get isInstalled() {\n return false\n },\n\n getDetails: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`getDetails`)\n }\n return null\n },\n getIsInstalled: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`getIsInstalled`)\n }\n return false\n },\n runningState: function getDetails() {\n if (arguments.length) {\n throw makeError.ErrorInInvocation(`runningState`)\n }\n return \'cannot_run\'\n }\n }\n utils.patchToStringNested(window.chrome.app)\n }',_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"utils => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n if ('csi' in window.chrome) {\n return // Nothing to do here\n }\n\n // Check that the Navigation Timing API v1 is available, we need that\n if (!window.performance || !window.performance.timing) {\n return\n }\n\n const { timing } = window.performance\n\n window.chrome.csi = function() {\n return {\n onloadT: timing.domContentLoadedEventEnd,\n startE: timing.navigationStart,\n pageT: Date.now() - timing.navigationStart,\n tran: 15 // Transition type or something\n }\n }\n utils.patchToString(window.chrome.csi)\n }",_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, { opts }) => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n if ('loadTimes' in window.chrome) {\n return // Nothing to do here\n }\n\n // Check that the Navigation Timing API v1 + v2 is available, we need that\n if (\n !window.performance ||\n !window.performance.timing ||\n !window.PerformancePaintTiming\n ) {\n return\n }\n\n const { performance } = window\n\n // Some stuff is not available on about:blank as it requires a navigation to occur,\n // let's harden the code to not fail then:\n const ntEntryFallback = {\n nextHopProtocol: 'h2',\n type: 'other'\n }\n\n // The API exposes some funky info regarding the connection\n const protocolInfo = {\n get connectionInfo() {\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ntEntry.nextHopProtocol\n },\n get npnNegotiatedProtocol() {\n // NPN is deprecated in favor of ALPN, but this implementation returns the\n // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n ? ntEntry.nextHopProtocol\n : 'unknown'\n },\n get navigationType() {\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ntEntry.type\n },\n get wasAlternateProtocolAvailable() {\n // The Alternate-Protocol header is deprecated in favor of Alt-Svc\n // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this\n // should always return false.\n return false\n },\n get wasFetchedViaSpdy() {\n // SPDY is deprecated in favor of HTTP/2, but this implementation returns\n // true for HTTP/2 or HTTP2+QUIC/39 as well.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n },\n get wasNpnNegotiated() {\n // NPN is deprecated in favor of ALPN, but this implementation returns true\n // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.\n const ntEntry =\n performance.getEntriesByType('navigation')[0] || ntEntryFallback\n return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)\n }\n }\n\n const { timing } = window.performance\n\n // Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3\n function toFixed(num, fixed) {\n var re = new RegExp('^-?\\\\d+(?:.\\\\d{0,' + (fixed || -1) + '})?')\n return num.toString().match(re)[0]\n }\n\n const timingInfo = {\n get firstPaintAfterLoadTime() {\n // This was never actually implemented and always returns 0.\n return 0\n },\n get requestTime() {\n return timing.navigationStart / 1000\n },\n get startLoadTime() {\n return timing.navigationStart / 1000\n },\n get commitLoadTime() {\n return timing.responseStart / 1000\n },\n get finishDocumentLoadTime() {\n return timing.domContentLoadedEventEnd / 1000\n },\n get finishLoadTime() {\n return timing.loadEventEnd / 1000\n },\n get firstPaintTime() {\n const fpEntry = performance.getEntriesByType('paint')[0] || {\n startTime: timing.loadEventEnd / 1000 // Fallback if no navigation occured (`about:blank`)\n }\n return toFixed(\n (fpEntry.startTime + performance.timeOrigin) / 1000,\n 3\n )\n }\n }\n\n window.chrome.loadTimes = function() {\n return {\n ...protocolInfo,\n ...timingInfo\n }\n }\n utils.patchToString(window.chrome.loadTimes)\n }",_args:[{opts:{}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, { opts, STATIC_DATA }) => {\n if (!window.chrome) {\n // Use the exact property descriptor found in headful Chrome\n // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`\n Object.defineProperty(window, 'chrome', {\n writable: true,\n enumerable: true,\n configurable: false, // note!\n value: {} // We'll extend that later\n })\n }\n\n // That means we're running headful and don't need to mock anything\n const existsAlready = 'runtime' in window.chrome\n // `chrome.runtime` is only exposed on secure origins\n const isNotSecure = !window.location.protocol.startsWith('https')\n if (existsAlready || (isNotSecure && !opts.runOnInsecureOrigins)) {\n return // Nothing to do here\n }\n\n window.chrome.runtime = {\n // There's a bunch of static data in that property which doesn't seem to change,\n // we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)`\n ...STATIC_DATA,\n // `chrome.runtime.id` is extension related and returns undefined in Chrome\n get id() {\n return undefined\n },\n // These two require more sophisticated mocks\n connect: null,\n sendMessage: null\n }\n\n const makeCustomRuntimeErrors = (preamble, method, extensionId) => ({\n NoMatchingSignature: new TypeError(\n preamble + `No matching signature.`\n ),\n MustSpecifyExtensionID: new TypeError(\n preamble +\n `${method} called from a webpage must specify an Extension ID (string) for its first argument.`\n ),\n InvalidExtensionID: new TypeError(\n preamble + `Invalid extension id: '${extensionId}'`\n )\n })\n\n // Valid Extension IDs are 32 characters in length and use the letter `a` to `p`:\n // https://source.chromium.org/chromium/chromium/src/+/master:components/crx_file/id_util.cc;drc=14a055ccb17e8c8d5d437fe080faba4c6f07beac;l=90\n const isValidExtensionID = str =>\n str.length === 32 && str.toLowerCase().match(/^[a-p]+$/)\n\n /** Mock `chrome.runtime.sendMessage` */\n const sendMessageHandler = {\n apply: function(target, ctx, args) {\n const [extensionId, options, responseCallback] = args || []\n\n // Define custom errors\n const errorPreamble = `Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback): `\n const Errors = makeCustomRuntimeErrors(\n errorPreamble,\n `chrome.runtime.sendMessage()`,\n extensionId\n )\n\n // Check if the call signature looks ok\n const noArguments = args.length === 0\n const tooManyArguments = args.length > 4\n const incorrectOptions = options && typeof options !== 'object'\n const incorrectResponseCallback =\n responseCallback && typeof responseCallback !== 'function'\n if (\n noArguments ||\n tooManyArguments ||\n incorrectOptions ||\n incorrectResponseCallback\n ) {\n throw Errors.NoMatchingSignature\n }\n\n // At least 2 arguments are required before we even validate the extension ID\n if (args.length < 2) {\n throw Errors.MustSpecifyExtensionID\n }\n\n // Now let's make sure we got a string as extension ID\n if (typeof extensionId !== 'string') {\n throw Errors.NoMatchingSignature\n }\n\n if (!isValidExtensionID(extensionId)) {\n throw Errors.InvalidExtensionID\n }\n\n return undefined // Normal behavior\n }\n }\n utils.mockWithProxy(\n window.chrome.runtime,\n 'sendMessage',\n function sendMessage() {},\n sendMessageHandler\n )\n\n /**\n * Mock `chrome.runtime.connect`\n *\n * @see https://developer.chrome.com/apps/runtime#method-connect\n */\n const connectHandler = {\n apply: function(target, ctx, args) {\n const [extensionId, connectInfo] = args || []\n\n // Define custom errors\n const errorPreamble = `Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): `\n const Errors = makeCustomRuntimeErrors(\n errorPreamble,\n `chrome.runtime.connect()`,\n extensionId\n )\n\n // Behavior differs a bit from sendMessage:\n const noArguments = args.length === 0\n const emptyStringArgument = args.length === 1 && extensionId === ''\n if (noArguments || emptyStringArgument) {\n throw Errors.MustSpecifyExtensionID\n }\n\n const tooManyArguments = args.length > 2\n const incorrectConnectInfoType =\n connectInfo && typeof connectInfo !== 'object'\n\n if (tooManyArguments || incorrectConnectInfoType) {\n throw Errors.NoMatchingSignature\n }\n\n const extensionIdIsString = typeof extensionId === 'string'\n if (extensionIdIsString && extensionId === '') {\n throw Errors.MustSpecifyExtensionID\n }\n if (extensionIdIsString && !isValidExtensionID(extensionId)) {\n throw Errors.InvalidExtensionID\n }\n\n // There's another edge-case here: extensionId is optional so we might find a connectInfo object as first param, which we need to validate\n const validateConnectInfo = ci => {\n // More than a first param connectInfo as been provided\n if (args.length > 1) {\n throw Errors.NoMatchingSignature\n }\n // An empty connectInfo has been provided\n if (Object.keys(ci).length === 0) {\n throw Errors.MustSpecifyExtensionID\n }\n // Loop over all connectInfo props an check them\n Object.entries(ci).forEach(([k, v]) => {\n const isExpected = ['name', 'includeTlsChannelId'].includes(k)\n if (!isExpected) {\n throw new TypeError(\n errorPreamble + `Unexpected property: '${k}'.`\n )\n }\n const MismatchError = (propName, expected, found) =>\n TypeError(\n errorPreamble +\n `Error at property '${propName}': Invalid type: expected ${expected}, found ${found}.`\n )\n if (k === 'name' && typeof v !== 'string') {\n throw MismatchError(k, 'string', typeof v)\n }\n if (k === 'includeTlsChannelId' && typeof v !== 'boolean') {\n throw MismatchError(k, 'boolean', typeof v)\n }\n })\n }\n if (typeof extensionId === 'object') {\n validateConnectInfo(extensionId)\n throw Errors.MustSpecifyExtensionID\n }\n\n // Unfortunately even when the connect fails Chrome will return an object with methods we need to mock as well\n return utils.patchToStringNested(makeConnectResponse())\n }\n }\n utils.mockWithProxy(\n window.chrome.runtime,\n 'connect',\n function connect() {},\n connectHandler\n )\n\n function makeConnectResponse() {\n const onSomething = () => ({\n addListener: function addListener() {},\n dispatch: function dispatch() {},\n hasListener: function hasListener() {},\n hasListeners: function hasListeners() {\n return false\n },\n removeListener: function removeListener() {}\n })\n\n const response = {\n name: '',\n sender: undefined,\n disconnect: function disconnect() {},\n onDisconnect: onSomething(),\n onMessage: onSomething(),\n postMessage: function postMessage() {\n if (!arguments.length) {\n throw new TypeError(`Insufficient number of arguments.`)\n }\n throw new Error(`Attempting to use a disconnected port object`)\n }\n }\n return response\n }\n }",_args:[{opts:{runOnInsecureOrigins:!1},STATIC_DATA:{OnInstalledReason:{CHROME_UPDATE:"chrome_update",INSTALL:"install",SHARED_MODULE_UPDATE:"shared_module_update",UPDATE:"update"},OnRestartRequiredReason:{APP_UPDATE:"app_update",OS_UPDATE:"os_update",PERIODIC:"periodic"},PlatformArch:{ARM:"arm",ARM64:"arm64",MIPS:"mips",MIPS64:"mips64",X86_32:"x86-32",X86_64:"x86-64"},PlatformNaclArch:{ARM:"arm",MIPS:"mips",MIPS64:"mips64",X86_32:"x86-32",X86_64:"x86-64"},PlatformOs:{ANDROID:"android",CROS:"cros",LINUX:"linux",MAC:"mac",OPENBSD:"openbsd",WIN:"win"},RequestUpdateCheckStatus:{NO_UPDATE:"no_update",THROTTLED:"throttled",UPDATE_AVAILABLE:"update_available"}}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"utils => {\n /**\n * Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing.\n *\n * @example\n * video/webm; codecs=\"vp8, vorbis\"\n * video/mp4; codecs=\"avc1.42E01E\"\n * audio/x-m4a;\n * audio/ogg; codecs=\"vorbis\"\n * @param {String} arg\n */\n const parseInput = arg => {\n const [mime, codecStr] = arg.trim().split(';')\n let codecs = []\n if (codecStr && codecStr.includes('codecs=\"')) {\n codecs = codecStr\n .trim()\n .replace(`codecs=\"`, '')\n .replace(`\"`, '')\n .trim()\n .split(',')\n .filter(x => !!x)\n .map(x => x.trim())\n }\n return {\n mime,\n codecStr,\n codecs\n }\n }\n\n const canPlayType = {\n // Intercept certain requests\n apply: function(target, ctx, args) {\n if (!args || !args.length) {\n return target.apply(ctx, args)\n }\n const { mime, codecs } = parseInput(args[0])\n // This specific mp4 codec is missing in Chromium\n if (mime === 'video/mp4') {\n if (codecs.includes('avc1.42E01E')) {\n return 'probably'\n }\n }\n // This mimetype is only supported if no codecs are specified\n if (mime === 'audio/x-m4a' && !codecs.length) {\n return 'maybe'\n }\n\n // This mimetype is only supported if no codecs are specified\n if (mime === 'audio/aac' && !codecs.length) {\n return 'probably'\n }\n // Everything else as usual\n return target.apply(ctx, args)\n }\n }\n\n /* global HTMLMediaElement */\n utils.replaceWithProxy(\n HTMLMediaElement.prototype,\n 'canPlayType',\n canPlayType\n )\n }",_args:[]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, { opts }) => {\n utils.replaceGetterWithProxy(\n Object.getPrototypeOf(navigator),\n 'hardwareConcurrency',\n utils.makeHandler().getterValue(opts.hardwareConcurrency)\n )\n }",_args:[{opts:{hardwareConcurrency:4}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, { opts }) => {\n const languages = opts.languages.length\n ? opts.languages\n : ['en-US', 'en']\n utils.replaceGetterWithProxy(\n Object.getPrototypeOf(navigator),\n 'languages',\n utils.makeHandler().getterValue(Object.freeze([...languages]))\n )\n }",_args:[{opts:{languages:[]}}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, opts) => {\n const isSecure = document.location.protocol.startsWith('https')\n\n // In headful on secure origins the permission should be \"default\", not \"denied\"\n if (isSecure) {\n utils.replaceGetterWithProxy(Notification, 'permission', {\n apply() {\n return 'default'\n }\n })\n }\n\n // Another weird behavior:\n // On insecure origins in headful the state is \"denied\",\n // whereas in headless it's \"prompt\"\n if (!isSecure) {\n const handler = {\n apply(target, ctx, args) {\n const param = (args || [])[0]\n\n const isNotifications =\n param && param.name && param.name === 'notifications'\n if (!isNotifications) {\n return utils.cache.Reflect.apply(...arguments)\n }\n\n return Promise.resolve(\n Object.setPrototypeOf(\n {\n state: 'denied',\n onchange: null\n },\n PermissionStatus.prototype\n )\n )\n }\n }\n // Note: Don't use `Object.getPrototypeOf` here\n utils.replaceWithProxy(Permissions.prototype, 'query', handler)\n }\n }",_args:[{}]}),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, { fns, data }) => {\n fns = utils.materializeFns(fns)\n\n // That means we're running headful\n const hasPlugins = 'plugins' in navigator && navigator.plugins.length\n if (hasPlugins) {\n return // nothing to do here\n }\n\n const mimeTypes = fns.generateMimeTypeArray(utils, fns)(data.mimeTypes)\n const plugins = fns.generatePluginArray(utils, fns)(data.plugins)\n\n // Plugin and MimeType cross-reference each other, let's do that now\n // Note: We're looping through `data.plugins` here, not the generated `plugins`\n for (const pluginData of data.plugins) {\n pluginData.__mimeTypes.forEach((type, index) => {\n plugins[pluginData.name][index] = mimeTypes[type]\n\n Object.defineProperty(plugins[pluginData.name], type, {\n value: mimeTypes[type],\n writable: false,\n enumerable: false, // Not enumerable\n configurable: true\n })\n Object.defineProperty(mimeTypes[type], 'enabledPlugin', {\n value:\n type === 'application/x-pnacl'\n ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks\n : new Proxy(plugins[pluginData.name], {}), // Prevent circular references\n writable: false,\n enumerable: false, // Important: `JSON.stringify(navigator.plugins)`\n configurable: true\n })\n })\n }\n\n const patchNavigator = (name, value) =>\n utils.replaceProperty(Object.getPrototypeOf(navigator), name, {\n get() {\n return value\n }\n })\n\n patchNavigator('mimeTypes', mimeTypes)\n patchNavigator('plugins', plugins)\n\n // All done\n }",_args:[{fns:{generateMimeTypeArray:"(utils, fns) => mimeTypesData => {\n return fns.generateMagicArray(utils, fns)(\n mimeTypesData,\n MimeTypeArray.prototype,\n MimeType.prototype,\n 'type'\n )\n}",generatePluginArray:"(utils, fns) => pluginsData => {\n return fns.generateMagicArray(utils, fns)(\n pluginsData,\n PluginArray.prototype,\n Plugin.prototype,\n 'name'\n )\n}",generateMagicArray:"(utils, fns) =>\n function(\n dataArray = [],\n proto = MimeTypeArray.prototype,\n itemProto = MimeType.prototype,\n itemMainProp = 'type'\n ) {\n // Quick helper to set props with the same descriptors vanilla is using\n const defineProp = (obj, prop, value) =>\n Object.defineProperty(obj, prop, {\n value,\n writable: false,\n enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)`\n configurable: true\n })\n\n // Loop over our fake data and construct items\n const makeItem = data => {\n const item = {}\n for (const prop of Object.keys(data)) {\n if (prop.startsWith('__')) {\n continue\n }\n defineProp(item, prop, data[prop])\n }\n return patchItem(item, data)\n }\n\n const patchItem = (item, data) => {\n let descriptor = Object.getOwnPropertyDescriptors(item)\n\n // Special case: Plugins have a magic length property which is not enumerable\n // e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes\n if (itemProto === Plugin.prototype) {\n descriptor = {\n ...descriptor,\n length: {\n value: data.__mimeTypes.length,\n writable: false,\n enumerable: false,\n configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`\n }\n }\n }\n\n // We need to spoof a specific `MimeType` or `Plugin` object\n const obj = Object.create(itemProto, descriptor)\n\n // Virtually all property keys are not enumerable in vanilla\n const blacklist = [...Object.keys(data), 'length', 'enabledPlugin']\n return new Proxy(obj, {\n ownKeys(target) {\n return Reflect.ownKeys(target).filter(k => !blacklist.includes(k))\n },\n getOwnPropertyDescriptor(target, prop) {\n if (blacklist.includes(prop)) {\n return undefined\n }\n return Reflect.getOwnPropertyDescriptor(target, prop)\n }\n })\n }\n\n const magicArray = []\n\n // Loop through our fake data and use that to create convincing entities\n dataArray.forEach(data => {\n magicArray.push(makeItem(data))\n })\n\n // Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards\n magicArray.forEach(entry => {\n defineProp(magicArray, entry[itemMainProp], entry)\n })\n\n // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)`\n const magicArrayObj = Object.create(proto, {\n ...Object.getOwnPropertyDescriptors(magicArray),\n\n // There's one ugly quirk we unfortunately need to take care of:\n // The `MimeTypeArray` prototype has an enumerable `length` property,\n // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`.\n // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap.\n length: {\n value: magicArray.length,\n writable: false,\n enumerable: false,\n configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length`\n }\n })\n\n // Generate our functional function mocks :-)\n const functionMocks = fns.generateFunctionMocks(utils)(\n proto,\n itemMainProp,\n magicArray\n )\n\n // We need to overlay our custom object with a JS Proxy\n const magicArrayObjProxy = new Proxy(magicArrayObj, {\n get(target, key = '') {\n // Redirect function calls to our custom proxied versions mocking the vanilla behavior\n if (key === 'item') {\n return functionMocks.item\n }\n if (key === 'namedItem') {\n return functionMocks.namedItem\n }\n if (proto === PluginArray.prototype && key === 'refresh') {\n return functionMocks.refresh\n }\n // Everything else can pass through as normal\n return utils.cache.Reflect.get(...arguments)\n },\n ownKeys(target) {\n // There are a couple of quirks where the original property demonstrates \"magical\" behavior that makes no sense\n // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length`\n // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly\n // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing\n // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing\n const keys = []\n const typeProps = magicArray.map(mt => mt[itemMainProp])\n typeProps.forEach((_, i) => keys.push(`${i}`))\n typeProps.forEach(propName => keys.push(propName))\n return keys\n },\n getOwnPropertyDescriptor(target, prop) {\n if (prop === 'length') {\n return undefined\n }\n return Reflect.getOwnPropertyDescriptor(target, prop)\n }\n })\n\n return magicArrayObjProxy\n }",generateFunctionMocks:"utils => (\n proto,\n itemMainProp,\n dataArray\n) => ({\n /** Returns the MimeType object with the specified index. */\n item: utils.createProxy(proto.item, {\n apply(target, ctx, args) {\n if (!args.length) {\n throw new TypeError(\n `Failed to execute 'item' on '${\n proto[Symbol.toStringTag]\n }': 1 argument required, but only 0 present.`\n )\n }\n // Special behavior alert:\n // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup\n // - If anything else than an integer (including as string) is provided it will return the first entry\n const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer\n // Note: Vanilla never returns `undefined`\n return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null\n }\n }),\n /** Returns the MimeType object with the specified name. */\n namedItem: utils.createProxy(proto.namedItem, {\n apply(target, ctx, args) {\n if (!args.length) {\n throw new TypeError(\n `Failed to execute 'namedItem' on '${\n proto[Symbol.toStringTag]\n }': 1 argument required, but only 0 present.`\n )\n }\n return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`!\n }\n }),\n /** Does nothing and shall return nothing */\n refresh: proto.refresh\n ? utils.createProxy(proto.refresh, {\n apply(target, ctx, args) {\n return undefined\n }\n })\n : undefined\n})"},data:{mimeTypes:[{type:"application/pdf",suffixes:"pdf",description:"",__pluginName:"Chrome PDF Viewer"},{type:"application/x-google-chrome-pdf",suffixes:"pdf",description:"Portable Document Format",__pluginName:"Chrome PDF Plugin"},{type:"application/x-nacl",suffixes:"",description:"Native Client Executable",__pluginName:"Native Client"},{type:"application/x-pnacl",suffixes:"",description:"Portable Native Client Executable",__pluginName:"Native Client"}],plugins:[{name:"Chrome PDF Plugin",filename:"internal-pdf-viewer",description:"Portable Document Format",__mimeTypes:["application/x-google-chrome-pdf"]},{name:"Chrome PDF Viewer",filename:"mhjfbmdgcfjbbpaeojofohoefgiehjai",description:"",__mimeTypes:["application/pdf"]},{name:"Native Client",filename:"internal-nacl-plugin",description:"",__mimeTypes:["application/x-nacl","application/x-pnacl"]}]}}]}),!1===navigator.webdriver||void 0===navigator.webdriver||delete Object.getPrototypeOf(navigator).webdriver,(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, opts) => {\n const getParameterProxyHandler = {\n apply: function(target, ctx, args) {\n const param = (args || [])[0]\n const result = utils.cache.Reflect.apply(target, ctx, args)\n // UNMASKED_VENDOR_WEBGL\n if (param === 37445) {\n return opts.vendor || 'Intel Inc.' // default in headless: Google Inc.\n }\n // UNMASKED_RENDERER_WEBGL\n if (param === 37446) {\n return opts.renderer || 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader\n }\n return result\n }\n }\n\n // There's more than one WebGL rendering context\n // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility\n // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter)\n const addProxy = (obj, propName) => {\n utils.replaceWithProxy(obj, propName, getParameterProxyHandler)\n }\n // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing:\n addProxy(WebGLRenderingContext.prototype, 'getParameter')\n addProxy(WebGL2RenderingContext.prototype, 'getParameter')\n }",_args:[{}]}),(()=>{try{if(window.outerWidth&&window.outerHeight)return;const n=85;window.outerWidth=window.innerWidth,window.outerHeight=window.innerHeight+n}catch(n){}})(),(({_utilsFns:_utilsFns,_mainFunction:_mainFunction,_args:_args})=>{const utils=Object.fromEntries(Object.entries(_utilsFns).map((([key,value])=>[key,eval(value)])));utils.init(),eval(_mainFunction)(utils,..._args)})({_utilsFns:{init:"() => {\n utils.preloadCache()\n}",stripProxyFromErrors:"(handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value')\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value')\n }\n return Reflect.setPrototypeOf(target, proto)\n }\n }\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler)\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].apply(this, arguments || [])\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ]\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n )\n }\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n')\n anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n )\n if (anchorIndex === -1) {\n return false // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n return stackArr.join('\\n')\n }\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n )\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false)\n throw err\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack)\n\n throw err // Re-throw our now sanitized error\n }\n }\n })\n return newHandler\n}",stripErrorWithAnchor:"(err, anchor) => {\n const stackArr = err.stack.split('\\n')\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor))\n if (anchorIndex === -1) {\n return err // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex)\n err.stack = stackArr.join('\\n')\n return err\n}",replaceProperty:"(obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n })\n}",preloadCache:"() => {\n if (utils.cache) {\n return\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n }\n}",makeNativeString:"(name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '')\n}",patchToString:"(obj, str = '') => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n // `toString` targeted at our proxied Object detected\n if (ctx === obj) {\n // We either return the optional string verbatim or derive the most desired result automatically\n return str || utils.makeNativeString(obj.name)\n }\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",patchToStringNested:"(obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString)\n}",redirectToString:"(proxyObj, originalObj) => {\n const handler = {\n apply: function (target, ctx) {\n // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n if (ctx === Function.prototype.toString) {\n return utils.makeNativeString('toString')\n }\n\n // `toString` targeted at our proxied Object detected\n if (ctx === proxyObj) {\n const fallback = () =>\n originalObj && originalObj.name\n ? utils.makeNativeString(originalObj.name)\n : utils.makeNativeString(proxyObj.name)\n\n // Return the toString representation of our original object if possible\n return originalObj + '' || fallback()\n }\n\n if (typeof ctx === 'undefined' || ctx === null) {\n return target.call(ctx)\n }\n\n // Check if the toString protype of the context is the same as the global prototype,\n // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n const hasSameProto = Object.getPrototypeOf(\n Function.prototype.toString\n ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins\n if (!hasSameProto) {\n // Pass the call on to the local Function.prototype.toString instead\n return ctx.toString()\n }\n\n return target.call(ctx)\n }\n }\n\n const toStringProxy = new Proxy(\n Function.prototype.toString,\n utils.stripProxyFromErrors(handler)\n )\n utils.replaceProperty(Function.prototype, 'toString', {\n value: toStringProxy\n })\n}",replaceWithProxy:"(obj, propName, handler) => {\n const originalObj = obj[propName]\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.redirectToString(proxyObj, originalObj)\n\n return true\n}",replaceGetterWithProxy:"(obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get\n const fnStr = fn.toString() // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { get: proxyObj })\n utils.patchToString(proxyObj, fnStr)\n\n return true\n}",replaceGetterSetter:"(obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName)\n const handler = { ...ownPropertyDescriptor }\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get\n handler.get = function() {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this))\n }\n utils.redirectToString(handler.get, nativeFn)\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set\n handler.set = function(newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this))\n }\n utils.redirectToString(handler.set, nativeFn)\n }\n\n Object.defineProperty(obj, propName, handler)\n}",mockWithProxy:"(obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n\n utils.replaceProperty(obj, propName, { value: proxyObj })\n utils.patchToString(proxyObj)\n\n return true\n}",createProxy:"(pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler))\n utils.patchToString(proxyObj)\n\n return proxyObj\n}",splitObjPath:"objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n})",replaceObjPathWithProxy:"(objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath)\n const obj = eval(objName) // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler)\n}",execRecursively:"(obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key])\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key])\n }\n }\n }\n }\n recurse(obj)\n return obj\n}",stringifyFns:"(fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val\n return obj\n }, {})\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n )\n}",materializeFns:"(fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)] // eslint-disable-line no-eval\n }\n })\n )\n}",makeHandler:"() => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments)\n return value\n }\n })\n})",arrayEquals:"(array1, array2) => {\n if (array1.length !== array2.length) {\n return false\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false\n }\n }\n return true\n}",memoize:"fn => {\n const cache = []\n return function(...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) })\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value\n }\n}"},_mainFunction:"(utils, opts) => {\n try {\n // Adds a contentWindow proxy to the provided iframe element\n const addContentWindowProxy = iframe => {\n const contentWindowProxy = {\n get(target, key) {\n // Now to the interesting part:\n // We actually make this thing behave like a regular iframe window,\n // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :)\n // That makes it possible for these assertions to be correct:\n // iframe.contentWindow.self === window.top // must be false\n if (key === 'self') {\n return this\n }\n // iframe.contentWindow.frameElement === iframe // must be true\n if (key === 'frameElement') {\n return iframe\n }\n // Intercept iframe.contentWindow[0] to hide the property 0 added by the proxy.\n if (key === '0') {\n return undefined\n }\n return Reflect.get(target, key)\n }\n }\n\n if (!iframe.contentWindow) {\n const proxy = new Proxy(window, contentWindowProxy)\n Object.defineProperty(iframe, 'contentWindow', {\n get() {\n return proxy\n },\n set(newValue) {\n return newValue // contentWindow is immutable\n },\n enumerable: true,\n configurable: false\n })\n }\n }\n\n // Handles iframe element creation, augments `srcdoc` property so we can intercept further\n const handleIframeCreation = (target, thisArg, args) => {\n const iframe = target.apply(thisArg, args)\n\n // We need to keep the originals around\n const _iframe = iframe\n const _srcdoc = _iframe.srcdoc\n\n // Add hook for the srcdoc property\n // We need to be very surgical here to not break other iframes by accident\n Object.defineProperty(iframe, 'srcdoc', {\n configurable: true, // Important, so we can reset this later\n get: function() {\n return _srcdoc\n },\n set: function(newValue) {\n addContentWindowProxy(this)\n // Reset property, the hook is only needed once\n Object.defineProperty(iframe, 'srcdoc', {\n configurable: false,\n writable: false,\n value: _srcdoc\n })\n _iframe.srcdoc = newValue\n }\n })\n return iframe\n }\n\n // Adds a hook to intercept iframe creation events\n const addIframeCreationSniffer = () => {\n /* global document */\n const createElementHandler = {\n // Make toString() native\n get(target, key) {\n return Reflect.get(target, key)\n },\n apply: function(target, thisArg, args) {\n const isIframe =\n args && args.length && `${args[0]}`.toLowerCase() === 'iframe'\n if (!isIframe) {\n // Everything as usual\n return target.apply(thisArg, args)\n } else {\n return handleIframeCreation(target, thisArg, args)\n }\n }\n }\n // All this just due to iframes with srcdoc bug\n utils.replaceWithProxy(\n document,\n 'createElement',\n createElementHandler\n )\n }\n\n // Let's go\n addIframeCreationSniffer()\n } catch (err) {\n // console.warn(err)\n }\n }",_args:[]}); \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..4d2c6bd --- /dev/null +++ b/main.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + + +import argparse +import asyncio +import os +import sys +import config +import db +import utils.date_format as date_format +from base.enums import Platform +from models import monitor_task_model +from utils.scheduler import SchedulerManager +from utils.utils import logger + + +def task_group(tasks): + groups = {} + for name, enum in Platform.__members__.items(): + groups[enum.value] = [] + for task in tasks: + if task.platform in groups: + groups[task.platform].append(task) + return list(groups.values()) + + +async def do_get_task_job(): + """ + 获取任务信息 + :return: + """ + await db.init() + tasks = await monitor_task_model.get_today_task() + if not tasks: + logger.info(F"没有获取到任务信息") + return + # 分组 + # groups = task_group(tasks) + # random.shuffle(groups) + schedular_manager = SchedulerManager() + logger.info(F"============================== 获取到{len(tasks)}条任务信息 ==============================") + schedular_manager.add_tasks(tasks, True) + + +def restart(): + os.execl(sys.executable, sys.executable, *sys.argv) + + +def load_arg_parse(): + """ + 解析启动参数 + :return: + """ + parse = argparse.ArgumentParser(description="抓取社媒新闻数据") + parse.add_argument("-a", "--active", help="启动脚本时 立即进行一次任务拉取", default='false') + args = parse.parse_args() + logger.info(F"启动参数: {args}") + return args + + +def clear_system_proxy(): + # 清除系统代理相关的环境变量 + os.environ.pop('http_proxy', None) + os.environ.pop('https_proxy', None) + os.environ.pop('ftp_proxy', None) + os.environ.pop('no_proxy', None) + + +if __name__ == '__main__': + try: + clear_system_proxy() + logger.info(F'启动成功 将在每天的{config.GET_TASK_TIME}拉取任务信息') + get_task_time = date_format.gen_job_datetime(config.GET_TASK_TIME) + manager = SchedulerManager() + # 启动定时任务 + manager.start() + # 添加拉取任务信息的任务 + manager.scheduler.add_job(do_get_task_job, 'cron', hour=get_task_time.hour, minute=get_task_time.minute) + manager.scheduler.add_job(restart, 'cron', hour=get_task_time.hour, minute=0) + # 参数检查 + args = load_arg_parse() + if args.active and args.active.lower() == 'true': + logger.info(F"立即执行一次任务拉取...") + asyncio.get_event_loop().run_until_complete(do_get_task_job()) + # 开启事件循环 + asyncio.get_event_loop().run_forever() + except KeyboardInterrupt: + sys.exit() diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/models/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/models/monitor_result_db.py b/models/monitor_result_db.py new file mode 100644 index 0000000..50017a6 --- /dev/null +++ b/models/monitor_result_db.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from typing import Optional, Iterable + +from tortoise import fields, BaseDBAsyncClient +from tortoise.models import Model +import utils.date_format as date_format + + +class MonitorResult(Model): + """ + 结果实体 + """ + id = fields.IntField(pk=True, autoincrement=True, description="ID") + keyword = fields.CharField(null=True, max_length=120, description="关键词") + title = fields.CharField(null=True, max_length=255, description="文章标题") + url = fields.CharField(null=True, max_length=500, description="文章地址") + publish_time = fields.BigIntField(null=True, max_length=20, description="发布时间") + platform = fields.CharField(null=True, max_length=20, description="平台") + gather_time = fields.CharField(null=True, description="设定采集时间", max_length=30) + content = fields.TextField(null=True, description="文章内容") + image = fields.CharField(null=True, max_length=255, description="结果截图") + is_del = fields.IntField(null=True, max_length=1, description="删除状态") + create_time = fields.BigIntField(null=True, max_length=20, description="创建时间") + update_time = fields.BigIntField(null=True, max_length=20, description="更新时间") + delete_time = fields.BigIntField(null=True, max_length=20, description="删除时间") + + class Meta: + table = "aux_monitor_result" + + def _pre_save( + self, + using_db: Optional[BaseDBAsyncClient] = None, + update_fields: Optional[Iterable[str]] = None, + ) -> None: + if not self.id: + self.create_time = date_format.timestamp() + self.update_time = date_format.timestamp() + diff --git a/models/monitor_result_model.py b/models/monitor_result_model.py new file mode 100644 index 0000000..fe2d7dc --- /dev/null +++ b/models/monitor_result_model.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +import config +from .monitor_result_db import MonitorResult +import utils.date_format as date_format +from .monitor_task_db import MonitorTask + + +def gen_result(task: MonitorTask, title, url, publish_time): + """ + 构建任务结果对象 + :param task: 任务对象 + :param title: 标题 + :param url: 地址 + :param publish_time: 发布时间 + :return: + """ + if isinstance(publish_time, str): + timestamp = date_format.timestamp() + publish_time = int(date_format.parse(publish_time).timestamp()) + module = MonitorResult(title=title, url=url, publish_time=publish_time, + is_del=1, + keyword=task.keyword, platform=task.platform, + gather_time=F"{task.gather_date} {task.setting_time}") + return module + + +async def save(results): + if config.RESULT_UNIQUE: + await save_unique(results) + else: + model = MonitorResult() + await model.bulk_create(results) + + +async def save_unique(results): + # 过滤列表中重复的结果 + unique_results = {} + for result in results: + key = (result.platform, result.keyword, result.title) + if key not in unique_results: + unique_results[key] = result + unique_results = list(unique_results.values()) + # 过滤数据库中重复的结果 + save_results = [] + model = MonitorResult() + for result in unique_results: + exist = await model.filter(platform=result.platform, keyword=result.keyword, title=result.title).exists() + if not exist: + save_results.append(result) + await model.bulk_create(save_results) diff --git a/models/monitor_task_db.py b/models/monitor_task_db.py new file mode 100644 index 0000000..4731ebf --- /dev/null +++ b/models/monitor_task_db.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +from typing import Optional, Iterable + +from tortoise import fields, BaseDBAsyncClient +from tortoise.models import Model +import utils.date_format as date_format + + +class MonitorTask(Model): + """ + 任务实体 + """ + id = fields.IntField(pk=True, autoincrement=True, description="ID") + keyword = fields.CharField(null=True, max_length=255, description="关键词") + lang = fields.CharField(null=True, max_length=50, description="语言") + platform = fields.CharField(null=True, max_length=30, description="媒体平台") + gather_date = fields.CharField(null=True, max_length=30, description="采集日期") + gather_time = fields.CharField(null=True, max_length=30, description="采集时间") + setting_time = fields.CharField(null=True, max_length=30, description="设定时间") + status = fields.IntField(null=True, max_length=1, description="任务状态 1 待执行 2 进行中 3 已完成") + create_time = fields.BigIntField(null=True, max_length=16, description="创建时间") + update_time = fields.BigIntField(null=True, max_length=16, description="更新时间") + + class Meta: + table = "aux_monitor_task" + + async def _pre_save( + self, + using_db: Optional[BaseDBAsyncClient] = None, + update_fields: Optional[Iterable[str]] = None, + ) -> None: + if not self.id: + self.create_time = date_format.timestamp() + self.update_time = date_format.timestamp() + diff --git a/models/monitor_task_model.py b/models/monitor_task_model.py new file mode 100644 index 0000000..9358435 --- /dev/null +++ b/models/monitor_task_model.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +from .monitor_task_db import MonitorTask +import utils.date_format as date_format +from base.enums import TaskStatus + + +async def get_today_task(): + """ + 获取当天的任务信息 + :return: + """ + # 当天日期 + today = date_format.gen_today_str() + task_model = MonitorTask() + result = await task_model.filter(gather_date=today, status=TaskStatus.WAITING.value).all() + return result + + +async def get_task(task_id): + """ + 获取指定id的任务信息 + :param task_id: + :return: + """ + task_model = MonitorTask() + return await task_model.get_or_none(id=task_id) + + +async def complete(task_id): + task_model = MonitorTask() + await task_model.filter(id=task_id).update(status=TaskStatus.COMPLETED.value, update_time=date_format.timestamp()) + + +async def running(task_id): + task_model = MonitorTask() + await task_model.filter(id=task_id).update(status=TaskStatus.RUNNING.value, update_time=date_format.timestamp()) + + +async def fail(task_id): + task_model = MonitorTask() + await task_model.filter(id=task_id).update(status=TaskStatus.FAIL.value, update_time=date_format.timestamp()) + + +if __name__ == '__main__': + get_today_task() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..98e2af4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +tortoise-orm~=0.19.0 +playwright~=1.42.0 +httpx~=0.27.0 +aiomysql~=0.2.0 +pymysql~=1.1.0 +python-dateutil~=2.9.0.post0 +APScheduler~=3.10.4 +yagmail~=0.15.293 +retry~=0.9.2 + diff --git a/spiders/__init__.py b/spiders/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/spiders/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/spiders/renmin/__init__.py b/spiders/renmin/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/spiders/renmin/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/spiders/renmin/client.py b/spiders/renmin/client.py new file mode 100644 index 0000000..cbe8544 --- /dev/null +++ b/spiders/renmin/client.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +import json +from typing import Dict +from urllib.parse import urlencode +import httpx +from httpx._exceptions import HTTPError, RequestError +from playwright.async_api import Page +from .exception import DataFetchError +import asyncio +import json +import utils.date_format as date_format +from utils.utils import count_characters +from utils.utils import logger +import utils.proxy as proxy +import config + + +class RenMinClient: + def __init__(self, + timeout=60, + *, + playwright_page: Page, + cookie_dict: Dict[str, str]): + self.timeout = timeout + self.headers = { + "Accept": "application/json, text/plain, */*", + "Accept-Encoding": "gzip, deflate", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Cookie": "__jsluid_h=103d2323e283c476b59b2fdd3b9a5371; sso_c=0; sfr=1", + "Host": "search.people.cn", + "Content-Length": "163", + "Content-Type": "application/json", + "Origin": "http://search.people.cn", + "Pragma": "no-cache", + "Referer": "http://search.people.cn/s?keyword=%E4%B9%A1%E6%9D%91%E6%8C%AF%E5%85%B4&st=0&_=1710919073824", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + self._host = "http://search.people.cn" + self.playwright_page = playwright_page + self.cookie_dict = cookie_dict + + async def request(self, method, url, **kwargs): + """ + 请求方法 + :param method: 请求方法 + :param url: 地址 + :param kwargs: 参数 + :return: 返回结果 + """ + # api代理 + proxies = proxy.get_ip().to_httpx_proxies() if config.API_PROXY else None + try: + async with httpx.AsyncClient(proxies=proxies) as client: + response = await client.request( + method, url, timeout=self.timeout, + **kwargs + ) + # 人民网504 是没有数据 + if response.status_code == 504: + # logger.error(F"[人民网]黑名单异常: [{method}]{url} 参数: {kwargs}") + # raise DataFetchError("黑名单异常", url, method, kwargs) + return {} + if not response.status_code == 200: + logger.error(F"[人民网]httpx异常[{response.status_code}]: [{method}]{url} 参数: {kwargs}") + raise DataFetchError("httpx异常", url, method, kwargs) + data: Dict = response.json() + if data.get("code") != "0": + raise DataFetchError(data.get("message", "未知错误"), url) + else: + return data.get("data", {}) + except HTTPError as e: + logger.error(F"[人民网]httpx异常: [{e.request.method}]{e.request.url} 参数: {kwargs}") + logger.error(F"[人民网]错误信息{str(e)}") + raise DataFetchError(str(e), url) + except Exception as e: + logger.error(F"[人民网]未知的请求方法异常: [{method}]{url} 参数: {kwargs}") + logger.error(F"[人民网]错误信息{str(e)}") + raise Exception(str(e)) + + async def get(self, uri: str, params=None) -> Dict: + """ + GET 请求方法 + :param uri: 请求地址 + :param params: 参数 + :return: 返回结果 + """ + final_uri = uri + if isinstance(params, dict): + final_uri = (f"{uri}?" + f"{urlencode(params)}") + return await self.request(method="GET", url=F"{self._host}{final_uri}", headers=self.headers) + + async def post(self, uri: str, data: dict) -> Dict: + """ + POST 请求方法 + :param uri: 请求地址 + :param data: 参数 + :return: 返回结果 + """ + json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) + return await self.request(method="POST", url=F"{self._host}{uri}", + data=json_str, headers=self.headers) + + async def search(self, keyword, cur_page): + """ + 搜索 + :param end: + :param start: + :param keyword: 关键词 + :param cur_page: 页码 + :param sort_field: 排序 # 0 时间倒序 1 时间正序 + :return: + """ + # 接口地址 + uri = '/search-platform/front/search' + get_param = { + 'key': keyword, + 'startTime': 0, + 'endTime': 0, + 'hasContent': True, + 'hasTitle': True, + 'isFuzzy': False, # 精准匹配 + 'limit': 10, + 'page': cur_page, + 'sortType': 0, + 'type': 0 + } + chinese, not_chinese = count_characters(keyword) + # 长度 = 127+ 汉字*3 + 其他*1 + # 关键字部分 + content_length = 126 + (chinese * 3) + not_chinese + 1 # 如果精准匹配是False 加一字节 + # 页码部分 + chinese, not_chinese = count_characters(cur_page) + content_length = content_length + not_chinese + + logger.info(F"[人民网]请求长度: {content_length}") + logger.info(F"[人民网]参数: {get_param}") + self.headers['Content-Length'] = str(content_length) + content = await self.post(uri, get_param) + if not content or not content.get('records'): + return [] + return content.get('records', []) + + +if __name__ == '__main__': + client = RenMinClient(playwright_page=None, cookie_dict={}) + start, end = date_format.today_timestamp_long() + asyncio.run(client.search('乡村发展', 1)) diff --git a/spiders/renmin/exception.py b/spiders/renmin/exception.py new file mode 100644 index 0000000..f6bc068 --- /dev/null +++ b/spiders/renmin/exception.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from httpx import RequestError + + +class DataFetchError(RequestError): + """未知异常""" + def __init__(self, message, url, method="GET", params=None): + self.message = message + self.url = url + self.method = method + self.params = params + + def __str__(self): + return self.message + + +class IPBlockError(RequestError): + """ip被封禁异常""" diff --git a/spiders/renmin/spider.py b/spiders/renmin/spider.py new file mode 100644 index 0000000..71b72bd --- /dev/null +++ b/spiders/renmin/spider.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +from playwright.async_api import async_playwright, Page, BrowserType, BrowserContext + +from base.base_spider import AbstractSpider +from typing import Dict, List, Optional, Tuple +from .client import RenMinClient +from utils.utils import logger, is_blank +from models.monitor_task_model import get_task, running, complete, fail +from models.monitor_result_model import gen_result, save +from base.enums import Platform +import utils.date_format as date_format +import os +import config +import uuid +from .exception import DataFetchError +import utils.mail as mail +import asyncio +from tortoise.transactions import in_transaction + + +class RenMinSpider(AbstractSpider): + """ + 人民网爬虫 + """ + client: RenMinClient # 请求对象 + context_page: Page # 浏览器页面上下文 + browser_context: BrowserContext # 浏览器上下文 + image_path: str + + def __init__(self): + self.index_url = "http://www.people.com.cn/" + self.platform = Platform.REN_MIN + self.image_path = None + self.retry = 0 # 自旋次数 + + def init_config(self): + super().init_config() + + async def start(self, task_id): + try: + async with in_transaction(): + await self.do_spider(task_id) + except DataFetchError as e: + logger.error(F"[人民网]任务ID: {task_id} 获取数据异常") + logger.error(F"[人民网]任务ID: {task_id} 异常信息: {str(e)}") + # 尝试自旋 + self.retry = self.retry + 1 + if self.retry > 3: + await fail(task_id) + logger.error(F"[人民网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "人民网", str(e)) + else: + logger.info(F"[人民网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + except Exception as e: + logger.error(F"[人民网]任务ID: {task_id} 爬虫异常") + logger.error(F"[人民网]任务ID: {task_id} 异常信息: {str(e)}") + # 切换代理ip并自旋 + # 尝试自旋 + self.retry = self.retry + 1 + if self.retry > 3: + await fail(task_id) + logger.error(F"[人民网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "人民网", str(e)) + else: + logger.info(F"[人民网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + + async def create_client(self) -> RenMinClient: + return RenMinClient(playwright_page=None, cookie_dict={}) + + async def launch_browser(self, + chromium: BrowserType, + playwright_proxy: Optional[Dict], + user_agent: Optional[Dict], + headless: bool = True): + """ + 启动一个浏览器上下文 + :param chromium: + :param headless: + :param self: 类型 + :param playwright_proxy: 代理 + :param user_agent: 用户标识 + :return: + """ + # 浏览器对象 + browser = await chromium.launch(proxy=playwright_proxy, headless=headless) + + # 浏览器上下文 + browser_context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=user_agent + ) + return browser_context + + async def do_search(self, task): + """ + 获取任务信息 + :return: + """ + start, end = date_format.today_timestamp_long() # 开始结束时间 + results = [] + cur_page = 1 + logger.info(F"[人民网]开始执行任务 ID: {task.id} 关键词: {task.keyword} 语言: {task.lang}") + self.client = await self.create_client() + while True: + logger.info(F"[人民网]开始获取搜索结果 关键词: {task.keyword} 页码: {cur_page}") + search_datas = await self.client.search(task.keyword, cur_page) + logger.info(F"[人民网]获取到{len(search_datas)}条搜索结果") + if not search_datas: + logger.info(F"[人民网]关键词: {task.keyword} 页码: {cur_page}没有搜索到数据") + break + index = -1 + for i, data in enumerate(search_datas): + # 找到一个不是今天的数据就结束 + if not date_format.is_today(date_format.timestamp2date(data.get("displayTime")).strftime("%Y-%m-%d")): + index = i + break + # 切割 + if index == -1: + # 搜索结果的最后一个依然是今天的 整个添加 + results = results + search_datas + # 翻到下一页 继续找 + cur_page = cur_page + 1 + else: + # 搜索结果中有不是今天的 切割一部分添加 + results = results + search_datas[:index] + # 结束本次搜索 + break + logger.info(F"[人民网]关键词:{task.keyword} 搜索结束 总页码: {cur_page} 总条数: {len(results)}") + return results + + async def cut_screen(self, url): + """ + 网页截图 + :param url: 地址 + :return: + """ + if not self.image_path: + image_path = config.IMAGE_PATH + if is_blank(image_path): + self.image_path = "./data" + if not os.path.exists(self.image_path): + os.makedirs(self.image_path) + save_path = F"{self.image_path}/{uuid.uuid4()}.png" + # 开始截图 + await self.context_page.goto(url) + await self.context_page.screenshot(path=save_path, full_page=True) + return save_path + + async def do_spider(self, task_id): + # 获取任务信息 + task = await get_task(task_id) + if not task: + logger.error(F"[人民网]任务ID: {task_id}不存在 任务结束") + return + logger.info(F"[人民网]任务ID: {task_id} 任务开始") + await running(task_id) + # 从api中获取数据 + search_datas = await self.do_search(task) + if not search_datas: + logger.info(F"[人民网]任务ID: {task_id} 关键词:{task.keyword} 未搜索到结果 任务结束") + await complete(task_id) + return + # 保存result实体 + results = [] + # 启动浏览器 + async with async_playwright() as playwright: + chromium = playwright.chromium + self.browser_context = await self.launch_browser(chromium, None, None, headless=True) + # 反反爬脚本 + await self.browser_context.add_init_script(path="lib/stealth.min.js") + self.context_page: Page = await self.browser_context.new_page() + # 构建结果实体 截图 + for data in search_datas: + result = gen_result(task, data.get("title"), data.get("url"), int(data.get("displayTime") / 1000)) + # img_path = await self.cut_screen(data.get("url")) + # result.image = img_path + results.append(result) + # logger.info(F"[人民网]标题: {data.get('title')} 截图文件名: {img_path}") + + # 结果落库 + await save(results) + logger.info(F"[人民网]任务ID: {task_id} 关键词: {task.keyword} 保存{len(results)}条数据 任务结束") + await complete(task_id) diff --git a/spiders/xinhua/__init__.py b/spiders/xinhua/__init__.py new file mode 100644 index 0000000..fbc1537 --- /dev/null +++ b/spiders/xinhua/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + + + diff --git a/spiders/xinhua/client.py b/spiders/xinhua/client.py new file mode 100644 index 0000000..7fcf2ce --- /dev/null +++ b/spiders/xinhua/client.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +import json +from typing import Dict +from urllib.parse import urlencode +from .exception import DataFetchError +import httpx +from playwright.async_api import Page +from httpx._exceptions import HTTPError +from utils.utils import logger +import asyncio +import utils.proxy as proxy +import config + + +class XinHuaClient: + def __init__(self, + timeout=10, + *, + headers: Dict[str, str], + playwright_page: Page, + cookie_dict: Dict[str, str]): + self.timeout = timeout + self.headers = headers + self._host = "https://so.news.cn/" + self.playwright_page = playwright_page + self.cookie_dict = cookie_dict + + async def request(self, method, url, **kwargs): + """ + 请求方法 + :param method: 请求方法 + :param url: 地址 + :param kwargs: 参数 + :return: 返回结果 + """ + # api代理 + proxies = proxy.get_ip().to_httpx_proxies() if config.API_PROXY else None + try: + async with httpx.AsyncClient(proxies=proxies) as client: + response = await client.request( + method, url, timeout=self.timeout, + **kwargs + ) + # 返回不正确的状态码 + if not response.status_code == 200: + logger.error(F"[新华网]httpx异常[{response.status_code}]: [{method}]{url} 参数: {kwargs}") + raise DataFetchError("httpx异常", url, method, kwargs) + # 返回正确的状态码 + data: Dict = response.json() + if data.get("code") != 200: + # 有特殊情况 敏感词会直接把content返回为没有找到相关稿件 + if data.get("content") == '没有找到相关稿件': + logger.warning(F"[新华网]触发敏感词 跳过请求 参数: {kwargs}") + return {} + raise DataFetchError(data.get("content", "API未知错误"), url, method, kwargs) + else: + return data.get("content", {}) + except HTTPError as e: + logger.error(F"[新华网]httpx异常: [{method}]{url} 参数: {kwargs}") + logger.error(F"[新华网]错误信息{str(e)}") + raise DataFetchError(str(e), url) + except Exception as e: + logger.error(F"[新华网]未知的请求方法异常: [{method}]{url} 参数: {kwargs}") + logger.error(F"[新华网]错误信息{str(e)}") + raise Exception(str(e)) + + async def get(self, uri: str, params=None) -> Dict: + """ + GET 请求方法 + :param uri: 请求地址 + :param params: 参数 + :return: 返回结果 + """ + final_uri = uri + if isinstance(params, dict): + final_uri = (f"{uri}?" + f"{urlencode(params)}") + return await self.request(method="GET", url=F"{self._host}{final_uri}", headers=self.headers) + + async def post(self, uri: str, data: dict) -> Dict: + """ + POST 请求方法 + :param uri: 请求地址 + :param data: 参数 + :return: 返回结果 + """ + json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) + return await self.request(method="POST", url=F"{self._host}{uri}", + data=json_str, headers=self.headers) + + async def search(self, keyword, cur_page, lang='cn', sort_field=0, search_fields=0): + """ + 搜索 + :param lang: + :param keyword: 关键词 + :param cur_page: 页码 + :param sort_field: 排序 0: 相关度 1: 时间 + :param search_fields: 搜索类型: 0: 全文 1: 标题 + :return: + """ + # 接口地址 + uri = '/getNews' + get_param = { + 'keyword': keyword, + 'curPage': cur_page, + 'sortField': sort_field, + 'searchFields': search_fields, + 'lang': lang + } + content = await self.get(uri, get_param) + if not content or not content.get('results'): + return [] + return content.get('results', []) diff --git a/spiders/xinhua/exception.py b/spiders/xinhua/exception.py new file mode 100644 index 0000000..f6bc068 --- /dev/null +++ b/spiders/xinhua/exception.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from httpx import RequestError + + +class DataFetchError(RequestError): + """未知异常""" + def __init__(self, message, url, method="GET", params=None): + self.message = message + self.url = url + self.method = method + self.params = params + + def __str__(self): + return self.message + + +class IPBlockError(RequestError): + """ip被封禁异常""" diff --git a/spiders/xinhua/spider.py b/spiders/xinhua/spider.py new file mode 100644 index 0000000..a647539 --- /dev/null +++ b/spiders/xinhua/spider.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- + +from playwright.async_api import async_playwright, Page, BrowserType, BrowserContext + +from base.base_spider import AbstractSpider +from typing import Dict, List, Optional, Tuple +from .client import XinHuaClient +from utils.utils import logger, is_blank +from models.monitor_task_model import get_task, running, complete, fail +from models.monitor_result_model import gen_result, save +from base.enums import Platform +import utils.date_format as date_format +import os +import config +import uuid +from .exception import DataFetchError +import asyncio +import utils.mail as mail +from tortoise.transactions import in_transaction + + +class XinHuaSpider(AbstractSpider): + """ + 新华网爬虫 + """ + client: XinHuaClient # 请求对象 + context_page: Page # 浏览器页面上下文 + browser_context: BrowserContext # 浏览器上下文 + image_path: str + + def __init__(self): + self.index_url = "http://www.xinhuanet.com/" + self.platform = Platform.XIN_HUA + self.image_path = None + self.retry = 0 # 自旋次数 + self.context_page = None + + def init_config(self): + super().init_config() + + async def start(self, task_id): + try: + async with in_transaction(): + await self.do_spider(task_id) + except DataFetchError as e: + logger.error(F"[新华网]任务ID: {task_id} 获取数据异常") + logger.error(F"[新华网]任务ID: {task_id} 异常信息: {str(e)}") + # 尝试自旋 + self.retry = self.retry + 1 + if self.retry > 3: + await fail(task_id) + logger.error(F"[新华网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "新华网", str(e)) + else: + logger.info(F"[新华网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + except Exception as e: + logger.error(F"[新华网]任务ID: {task_id} 爬虫异常") + logger.error(F"[新华网]任务ID: {task_id} 异常信息: {str(e)}") + # 尝试自旋 + self.retry = self.retry + 1 + await fail(task_id) + if self.retry > 3: + logger.error(F"[新华网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "新华网", str(e)) + else: + logger.info(F"[新华网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + + async def create_xinhua_client(self, httpx_proxy: Optional[str]) -> XinHuaClient: + # 请求头 + headers = { + "Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Cache-Control": "no-cache", "Connection": "keep-alive", + "Cookie": "org.springframework.web.servlet.i18n.CookieLocaleResolver.LOCALE=zh_CN; wdcid=7af5eba7b2f8b44b; arialoadData=false; acw_tc=2760778017108394678246790e1403779a009cc2c5fe412f126407bf171637", + "Host": "so.news.cn", "Pragma": "no-cache", "Referer": "https://so.news.cn/", "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "X-Requested-With": "XMLHttpRequest", + "sec-ch-ua": "\"Chromium\";v=\"122\", \"Not(A:Brand\";v=\"24\", \"Google Chrome\";v=\"122\"", + "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Windows\""} + client = XinHuaClient(headers=headers, cookie_dict=None, playwright_page=self.context_page) + return client + + async def launch_browser(self, + chromium: BrowserType, + playwright_proxy: Optional[Dict], + user_agent: Optional[Dict], + headless: bool = True): + """ + 启动一个浏览器上下文 + :param chromium: + :param headless: + :param self: 类型 + :param playwright_proxy: 代理 + :param user_agent: 用户标识 + :return: + """ + # 浏览器对象 + browser = await chromium.launch(proxy=playwright_proxy, headless=headless) + + # 浏览器上下文 + browser_context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=user_agent + ) + return browser_context + + async def do_search(self, task): + """ + 获取任务信息 + :return: + """ + results = [] + cur_page = 1 + logger.info(F"[新华网]开始执行任务 ID: {task.id} 关键词: {task.keyword} 语言: {task.lang}") + self.client = await self.create_xinhua_client(None) + while True: + logger.info(F"[新华网]开始获取搜索结果 关键词: {task.keyword} 页码: {cur_page}") + search_datas = await self.client.search(keyword=task.keyword, cur_page=cur_page, lang=task.lang) + logger.info(F"[新华网]获取到{len(search_datas)}条搜索结果") + if not search_datas: + logger.info(F"[新华网]关键词: {task.keyword} 页码: {cur_page}没有搜索到数据") + break + index = -1 + for i, data in enumerate(search_datas): + # 找到一个不是今天的数据就结束 + if not date_format.is_today(data.get("pubtime")): + index = i + break + # 如果全都是今天的 就翻页 + if index == -1: + # 搜索结果的最后一个依然是今天的 整个添加 + results = results + search_datas + # 翻到下一页 继续找 + cur_page = cur_page + 1 + else: + # 搜索结果中有不是今天的 切割一部分添加 + results = results + search_datas[:index] + # 结束本次搜索 + break + logger.info(F"[新华网]关键词: {task.keyword} 搜索结束 总页码: {cur_page} 总条数: {len(results)}") + return results + + async def cut_screen(self, url): + """ + 网页截图 + :param url: 地址 + :return: + """ + if not self.image_path: + image_path = config.IMAGE_PATH + if is_blank(image_path): + self.image_path = "./data" + if not os.path.exists(self.image_path): + os.makedirs(self.image_path) + save_path = F"{self.image_path}/{uuid.uuid4()}.png" + # 开始截图 + await self.context_page.goto(url) + await self.context_page.screenshot(path=save_path, full_page=True) + return save_path + + async def do_spider(self, task_id): + # 获取任务信息 + task = await get_task(task_id) + if not task: + logger.error(F"[新华网]任务ID: {task_id}不存在 任务结束") + return + logger.info(F"[新华网]任务ID: {task_id} 任务开始") + await running(task_id) + # 从api中获取数据 + search_datas = await self.do_search(task) + if not search_datas: + logger.info(F"[新华网]任务ID: {task_id} 关键词:{task.keyword} 未搜索到结果 任务结束") + await complete(task_id) + return + # 保存result实体 + results = [] + # 启动浏览器 + async with async_playwright() as playwright: + chromium = playwright.chromium + self.browser_context = await self.launch_browser(chromium, None, None, headless=True) + # 反反爬脚本 + await self.browser_context.add_init_script(path="lib/stealth.min.js") + self.context_page: Page = await self.browser_context.new_page() + + # 构建结果实体 截图 + for data in search_datas: + result = gen_result(task, data.get("title"), data.get("url"), data.get("pubtime")) + # img_path = await self.cut_screen(data.get("url")) + # result.image = img_path + results.append(result) + # logger.info(F"[新华网]标题: {data.get('title')} 截图文件名: {img_path}") + + # 结果落库 + await save(results) + logger.info(F"[新华网]任务ID: {task_id} 关键词: {task.keyword} 保存{len(results)}条数据 任务结束") + await complete(task_id) + diff --git a/spiders/yang_shi/__init__.py b/spiders/yang_shi/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/spiders/yang_shi/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/spiders/yang_shi/client.py b/spiders/yang_shi/client.py new file mode 100644 index 0000000..f1ff22b --- /dev/null +++ b/spiders/yang_shi/client.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- + +import json +from typing import Dict +from urllib.parse import urlencode +import httpx +from playwright.async_api import Page +from .exception import DataFetchError +import asyncio +import json +import utils.date_format as date_format +from utils.utils import count_characters +from playwright.async_api import async_playwright +import asyncio +from utils.utils import logger + + +class YangShiClient: + def __init__(self, + timeout=60, + proxies=None, + *, + playwright_page: Page, + cookie_dict: Dict[str, str]): + self.proxies = proxies + self.timeout = timeout + self.headers = { + "Accept": "application/json, text/plain, */*", + "Accept-Encoding": "gzip, deflate", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Cookie": "__jsluid_h=103d2323e283c476b59b2fdd3b9a5371; sso_c=0; sfr=1", + "Host": "search.people.cn", + "Content-Length": "163", + "Content-Type": "application/json", + "Origin": "http://search.people.cn", + "Pragma": "no-cache", + "Referer": "http://search.people.cn/s?keyword=%E4%B9%A1%E6%9D%91%E6%8C%AF%E5%85%B4&st=0&_=1710919073824", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + } + self._host = "https://search.cctv.com/" + self.playwright_page = playwright_page + self.cookie_dict = cookie_dict + + async def request(self, method, url, **kwargs): + """ + 请求方法 + :param method: 请求方法 + :param url: 地址 + :param kwargs: 参数 + :return: 返回结果 + """ + async with httpx.AsyncClient(proxies=self.proxies) as client: + response = await client.request( + method, url, timeout=self.timeout, + **kwargs + ) + data: Dict = response.json() + if data.get("code") != "0": + raise DataFetchError(data.get("message", "未知错误")) + else: + return data.get("data", {}) + + async def get(self, uri: str, params=None) -> Dict: + """ + GET 请求方法 + :param uri: 请求地址 + :param params: 参数 + :return: 返回结果 + """ + final_uri = uri + if isinstance(params, dict): + final_uri = (f"{uri}?" + f"{urlencode(params)}") + return await self.request(method="GET", url=F"{self._host}{final_uri}", headers=self.headers) + + async def post(self, uri: str, data: dict) -> Dict: + """ + POST 请求方法 + :param uri: 请求地址 + :param data: 参数 + :return: 返回结果 + """ + json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) + return await self.request(method="POST", url=F"{self._host}{uri}", + data=json_str, headers=self.headers) + + async def search(self, keyword, cur_page): + """ + 搜索 + :param keyword: 关键词 + :param cur_page: 页码 + :return: + """ + # 接口地址 + uri = F"/search.php?qtext={keyword}&page={cur_page}&type=web&sort=date&datepid=1&channel=&vtime=-1&is_search=1" + full_url = F"{self._host}{uri}" + try: + await self.playwright_page.goto(full_url) + results = [] + # 选择每一个结果元素 + elements = await self.playwright_page.query_selector_all("div.tright") + for element in elements: + title = "" + url = "" + publish_time = "" + # 标题元素 + tit = await element.query_selector(".tit") + if tit: + # 标题下面的链接 + span = await tit.query_selector("span") + url = await span.get_attribute("lanmu1") + + # 存放标题的a标签 + tit_a = await span.query_selector("a") + if tit_a: + title = await tit_a.inner_text() + # 发布时间元素 + tim = await element.query_selector(".src-tim .tim") + if tim: + tim_text = await tim.inner_text() + publish_time = tim_text.split("：")[1] + # 保存数据 + results.append({ + "keyword": keyword, + "title": title, + "url": url, + "publish_time": publish_time + }) + return results + except Exception as e: + logger.error(F"[央视网]搜索方法异常: 关键词: {keyword} 页码: {cur_page} {full_url}") + logger.error(F"[央视网]错误信息: {str(e)}") + raise DataFetchError(str(e), full_url) + + +async def run(): + async with async_playwright() as playwright: + # 启动浏览器 + async with async_playwright() as playwright: + chromium = playwright.chromium + browser = await chromium.launch(headless=False) + # 浏览器上下文 + browser_context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent="" + ) + # 反反爬脚本 + await browser_context.add_init_script(path="../../lib/stealth.min.js") + context_page: Page = await browser_context.new_page() + + # 创建对象 + client = YangShiClient(playwright_page=context_page, cookie_dict={}) + result = await client.search("医保", 1) + print(result) + + +if __name__ == '__main__': + asyncio.get_event_loop().run_until_complete(run()) diff --git a/spiders/yang_shi/exception.py b/spiders/yang_shi/exception.py new file mode 100644 index 0000000..f6bc068 --- /dev/null +++ b/spiders/yang_shi/exception.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from httpx import RequestError + + +class DataFetchError(RequestError): + """未知异常""" + def __init__(self, message, url, method="GET", params=None): + self.message = message + self.url = url + self.method = method + self.params = params + + def __str__(self): + return self.message + + +class IPBlockError(RequestError): + """ip被封禁异常""" diff --git a/spiders/yang_shi/spider.py b/spiders/yang_shi/spider.py new file mode 100644 index 0000000..0e54bda --- /dev/null +++ b/spiders/yang_shi/spider.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- + +from playwright.async_api import async_playwright, Page, BrowserType, BrowserContext + +from base.base_spider import AbstractSpider +from typing import Dict, List, Optional, Tuple +from .client import YangShiClient +from utils.utils import logger, is_blank +from models.monitor_task_model import get_task, running, complete, fail +from models.monitor_result_model import gen_result, save +from base.enums import Platform +import utils.date_format as date_format +import os +import config +import uuid +from .exception import DataFetchError +import utils.mail as mail +import asyncio +from tortoise.transactions import in_transaction + + +class YangShiSpider(AbstractSpider): + """ + 央视网爬虫 + """ + client: YangShiClient # 请求对象 + context_page: Page # 浏览器页面上下文 + browser_context: BrowserContext # 浏览器上下文 + image_path: str + + def __init__(self): + self.index_url = "https://tv.cctv.com/" + self.platform = Platform.YANG_SHI + self.image_path = None + self.retry = 0 # 自旋次数 + + def init_config(self): + super().init_config() + + async def start(self, task_id): + try: + async with in_transaction(): + await self.do_spider(task_id) + except DataFetchError as e: + logger.error(F"[央视网]任务ID: {task_id} 获取数据异常") + logger.error(F"[央视网]任务ID: {task_id} 异常信息: {str(e)}") + # 尝试自旋 + self.retry = self.retry + 1 + if self.retry > 3: + await fail(task_id) + logger.error(F"[央视网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "央视网", str(e)) + else: + logger.info(F"[央视网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + except Exception as e: + logger.error(F"[央视网]任务ID: {task_id} 爬虫异常") + logger.error(F"[央视网]任务ID: {task_id} 异常信息: {str(e)}") + # 切换代理ip并自旋 + logger.error(F"[央视网]任务ID: {task_id} 获取数据异常") + logger.error(F"[央视网]任务ID: {task_id} 异常信息: {str(e)}") + # 尝试自旋 + self.retry = self.retry + 1 + if self.retry > 3: + await fail(task_id) + logger.error(F"[央视网]任务ID: {task_id} 重试达到最大次数 即将发送告警邮件") + await mail.send_post_mail(task_id, "央视网", str(e)) + else: + logger.info(F"[央视网]任务ID: {task_id} 20秒后进行第{self.retry}次重试") + await asyncio.sleep(20) + await self.do_spider(task_id) + + async def create_client(self, httpx_proxy: Optional[str]) -> YangShiClient: + # 请求头 + client = YangShiClient(proxies=httpx_proxy, cookie_dict={}, playwright_page=self.context_page) + return client + + async def launch_browser(self, + chromium: BrowserType, + playwright_proxy: Optional[Dict], + user_agent: Optional[Dict], + headless: bool = True): + """ + 启动一个浏览器上下文 + :param chromium: + :param headless: + :param self: 类型 + :param playwright_proxy: 代理 + :param user_agent: 用户标识 + :return: + """ + # 浏览器对象 + browser = await chromium.launch(proxy=playwright_proxy, headless=headless) + + # 浏览器上下文 + browser_context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=user_agent + ) + # 反反爬脚本 + await browser_context.add_init_script(path="lib/stealth.min.js") + context_page = await browser_context.new_page() + return browser_context, context_page + + async def do_search(self, task): + """ + 获取任务信息 + :return: + """ + results = [] + cur_page = 1 + logger.info(F"[央视网]开始执行任务 ID: {task.id} 关键词: {task.keyword} 语言: {task.lang}") + self.client = await self.create_client(None) + while True: + logger.info(F"[央视网]任务ID: {task.id} 开始获取搜索结果 关键词: {task.keyword} 页码: {cur_page}") + search_datas = await self.client.search(keyword=task.keyword, cur_page=cur_page) + logger.info(F"[央视网]任务ID: {task.id} 获取到{len(search_datas)}条搜索结果") + if not search_datas: + logger.info(F"[央视网]任务ID: {task.id} 关键词: {task.keyword} 页码: {cur_page}没有搜索到数据") + break + index = -1 + for i, data in enumerate(search_datas): + # 找到一个不是今天的数据就结束 + if not date_format.is_today(data.get("publish_time")): + index = i + break + # 切割 + if index == -1: + # 搜索结果的最后一个依然是今天的 整个添加 + results = results + search_datas + # 翻到下一页 继续找 + cur_page = cur_page + 1 + else: + # 搜索结果中有不是今天的 切割一部分添加 + results = results + search_datas[:index] + # 结束本次搜索 + break + logger.info(F"[央视网]任务ID: {task.id} 关键词: {task.keyword} 搜索结束 总页码: {cur_page} 总条数: {len(results)}") + return results + + async def cut_screen(self, url): + """ + 网页截图 + :param url: 地址 + :return: + """ + if not self.image_path: + image_path = config.IMAGE_PATH + if is_blank(image_path): + self.image_path = "./data" + if not os.path.exists(self.image_path): + os.makedirs(self.image_path) + save_path = F"{self.image_path}/{uuid.uuid4()}.png" + # 开始截图 + await self.context_page.goto(url) + await self.context_page.screenshot(path=save_path, full_page=True) + return save_path + + async def do_spider(self, task_id): + # 获取任务信息 + task = await get_task(task_id) + if not task: + logger.error(F"[央视网]任务ID: {task_id}不存在 任务结束") + return + logger.info(F"[央视网]任务ID: {task_id} 任务开始") + await running(task_id) + results = [] + # 启动浏览器 + async with async_playwright() as playwright: + chromium = playwright.chromium + self.browser_context, self.context_page = await self.launch_browser(chromium, None, None, headless=True) + # 创建请求客户端 + search_datas = await self.do_search(task) + # 构建结果实体 截图 + for data in search_datas: + result = gen_result(task, data.get("title"), data.get("url"), data.get("publish_time")) + # img_path = await self.cut_screen(data.get("url")) + # result.image = img_path + results.append(result) + # logger.info(F"[央视网] 任务ID: {task_id} 标题: {data.get('title')} 截图文件名: {img_path}") + # 结果落库 + await save(results) + logger.info(F"[央视网] 任务ID: {task_id} 关键词: {task.keyword} 保存{len(results)}条数据 任务结束") + await complete(task_id) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/utils/date_format.py b/utils/date_format.py new file mode 100644 index 0000000..cbc1fda --- /dev/null +++ b/utils/date_format.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +from dateutil.parser import parse +from dateutil.relativedelta import relativedelta +from datetime import datetime, timedelta +import time + + +def gen_job_datetime(time_str, date_str=''): + """ + 生成任务的时间和日期 + :param time_str: 时间 + :param date_str: 日期 + :return: + """ + if not time_str: + return None + return parse(date_str + time_str) + + +def gen_today_str(): + today = datetime.today() + return today.strftime("%Y-%m-%d") + + +def timestamp(): + return int(time.time()) + + +def is_today(date_str): + publish_date = parse(date_str).date() # 获取日期部分，忽略时间部分 + today = datetime.today().date() # 获取今天的日期，忽略时间部分 + # 检查日期是否相等 + return publish_date == today + + +def timestamp2date(timestamp_long): + d = datetime.utcfromtimestamp(float(timestamp_long / 1000)) + return d + + +def today_timestamp_long(): + """ + 获取今天开始和结束的毫秒时间戳 + :return: + """ + start = parse("00:00") + end = start + timedelta(days=1) + return start.timestamp() * 1000, end.timestamp() * 1000 + + +def parse_time(time_str): + return datetime.strptime(time_str, '%H:%M').time() + + +def eq_time(time1: str, time2: str): + time1 = datetime.strptime(time1, '%H:%M').time() + time2 = datetime.strptime(time2, '%H:%M').time() + + today = datetime.today().date() + time1 = datetime.combine(today, time1) + time2 = datetime.combine(today, time2) + + return time1 == time2 + + +def ge_time(time1: str, time2: str): + """ + 比较time1是否大于等于time2 + :param time1: + :param time2: + :return: + """ + time1 = datetime.strptime(time1, '%H:%M').time() + time2 = datetime.strptime(time2, '%H:%M').time() + + today = datetime.today().date() + time1 = datetime.combine(today, time1) + time2 = datetime.combine(today, time2) + + return time1 >= time2 + + +def lt_time(time1: str, time2: str): + """ + 比较time1是否小于time2 + :param time1: + :param time2: + :return: + """ + time1 = datetime.strptime(time1, '%H:%M').time() + time2 = datetime.strptime(time2, '%H:%M').time() + + today = datetime.today().date() + time1 = datetime.combine(today, time1) + time2 = datetime.combine(today, time2) + + return time1 < time2 + + +if __name__ == '__main__': + print(lt_time("18:52", "23:55")) diff --git a/utils/mail.py b/utils/mail.py new file mode 100644 index 0000000..9984340 --- /dev/null +++ b/utils/mail.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +import yagmail +import config +from utils.utils import logger +import inspect +from datetime import datetime +from models.monitor_task_model import get_task, complete + + +async def send_post_mail(task_id, name, message="出现异常"): + keyword = "" + task = await get_task(task_id) + if not task: + message = F"不存在的任务ID: {task_id}" + else: + keyword = task.keyword + + mail_server = None + try: + mail_server = yagmail.SMTP(user=config.SMTP_USER, password=config.SMTP_PASSWORD, host=config.SMTP_HOST) + except Exception as e: + logger.error("[邮件]初始化失败 请检查邮件配置") + return + to = [] + if not config.POST_EMAIL: + logger.warn("[邮件]未配置用于接收邮件上报的邮箱地址 邮件上报被取消") + return + if isinstance(config.POST_EMAIL, str): + to.append(config.POST_EMAIL) + elif isinstance(config.POST_EMAIL, list): + to = config.POST_EMAIL + else: + logger.warn("[邮件]未配置用于接收邮件上报的邮箱地址 邮件上报被取消") + return + + title = F"HuoSpider上报邮件: {name}数据获取异常" + content = F""" + 异常任务ID: {task_id} + 异常站点: {name} + 关键词: {keyword} + 异常信息: {message} + 上报时间: {datetime.today().strftime("%Y-%m-%d %H:%M:%S")} + """ + inspect.cleandoc(content) + mail_server.send(to, title, content) + mail_server.close() + logger.info(f"[邮件]任务ID: {task_id} 异常上报邮件发送成功 收件人: {to}") + + +async def test(task_id, name, message="出现异常"): + keyword = "测试" + + mail_server = None + try: + mail_server = yagmail.SMTP(user=config.SMTP_USER, password=config.SMTP_PASSWORD, host=config.SMTP_HOST) + except Exception as e: + logger.error("[邮件]初始化失败 请检查邮件配置") + return + to = [] + if not config.POST_EMAIL: + logger.warn("[邮件]未配置用于接收邮件上报的邮箱地址 邮件上报被取消") + return + if isinstance(config.POST_EMAIL, str): + to.append(config.POST_EMAIL) + elif isinstance(config.POST_EMAIL, list): + to = config.POST_EMAIL + else: + logger.warn("[邮件]未配置用于接收邮件上报的邮箱地址 邮件上报被取消") + return + + title = F"HuoSpider上报邮件: {name}数据获取异常" + content = F""" + 异常任务ID: {task_id} + 异常站点: {name} + 关键词: {keyword} + 异常信息: {message} + 上报时间: {datetime.today().strftime("%Y-%m-%d %H:%M:%S")} + """ + inspect.cleandoc(content) + mail_server.send(to, title, content) + mail_server.close() + logger.info(f"[邮件]任务ID: {task_id} 异常上报邮件发送成功 收件人: {to}") diff --git a/utils/proxy.py b/utils/proxy.py new file mode 100644 index 0000000..ad3575f --- /dev/null +++ b/utils/proxy.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +from datetime import datetime, timedelta + +import httpx +import retry +from dateutil.parser import parse + +import config +from utils.utils import logger + +ip_pool = [] + + +# 代理池 + +class ProxyIp: + ip: str + port: int + expire: datetime + city: str + isp: str + + def __init__(self, ip, port, expire, city='未知', isp='未知'): + self.ip = ip + self.port = port + self.expire = parse(expire) + self.city = city + self.isp = isp + + def __str__(self): + return F"({self.city}-{self.isp}){self.ip}:{self.port} 过期时间:{self.expire}" + + def is_expire(self): + now = datetime.now() + expire = self.expire - timedelta(seconds=20) + return expire < now + + def to_httpx_proxies(self): + return {"http://": F"http://{self.ip}:{self.port}"} + + +class ProxyError(Exception): + def __init__(self, message, code=-1000): + self.code = code + self.message = message + + def __str__(self): + return F"错误码: {self.code} 错误消息: {self.message}" + + +@retry.retry(exceptions=ProxyError, tries=3, delay=2, backoff=2) +def add_ip(count=1) -> ProxyIp: + """ + 向ip池中添加一个代理ip对象 + :param count: 添加的数量 默认为1 + :return: + """ + url = "http://api.tianqiip.com/getip" + params = { + "secret": config.PROXY_SECRET, # 密钥 + "sign": config.PROXY_SIGN, # 签名 + "num": count, # 数量 + "type": "json", # 返回类型 + "port": 1, # 协议 + "time": 3, # 时长三分钟 + "ts": 1, # 显示过期时间 + "mr": 1, # 去重 + "cs": 1, # 显示位置 + "ys": 1 # 显示运营商 + } + ips = [] + result: dict = httpx.get(url, params=params, proxies={}).json() + if not result['code'] == 1000: + logger.error("[IP池]API获取代理IP失败") + raise ProxyError(result['code'], result['msg']) + for data in result["data"]: + ip = ProxyIp(data['ip'], data['port'], data['expire'], city=data['city'], isp=data['isp']) + ip_pool.append(ip) + ips.append(ip) + logger.info(F"[IP池]新增代理IP {str(ip)}") + return ips[0] + + +def del_ip(index): + if index > len(ip_pool) - 1: + return + logger.error(f"[IP池]代理IP被删除: {ip_pool[index]}") + del ip_pool[index] + + +def get_ip(cache=True) -> ProxyIp: + """ + 获取一个代理ip对象 + :param cache: 使用缓存 + :return: + """ + if not cache: + # 不使用缓存时 请求一个新的ip并放入池中 然后获取该ip + return add_ip() + # 从缓存中获取一个有效的ip + if not ip_pool: + return add_ip() + cur_ip = None + for index, ip in enumerate(ip_pool): + if not ip.is_expire(): + # 没过期 返回 + cur_ip = ip + break + if not cur_ip: + return add_ip() + logger.info(f"[IP池]从IP池中获取到代理IP: {cur_ip}") + return cur_ip diff --git a/utils/scheduler.py b/utils/scheduler.py new file mode 100644 index 0000000..226a21c --- /dev/null +++ b/utils/scheduler.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +import datetime +import random + +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from typing import Dict, List + +import config +from models.monitor_task_db import MonitorTask +from spiders.xinhua.spider import XinHuaSpider +from spiders.renmin.spider import RenMinSpider +from spiders.yang_shi.spider import YangShiSpider +from utils.utils import logger +from base.base_spider import AbstractSpider +import utils.date_format as date_format +from base.enums import Platform +import logging +from datetime import timedelta +import copy + + +def singleton(cls): + instances = {} + + def getinstance(): + if cls not in instances: + instances[cls] = cls() + return instances[cls] + + return getinstance + + +@singleton +class SchedulerManager: + scheduler: AsyncIOScheduler + + def __init__(self): + # 调整调度器日志等级 + ap_logger = logging.getLogger('apscheduler') + ap_logger.setLevel(logging.WARNING) + self.scheduler = AsyncIOScheduler() + + def get_scheduler(self): + """ + 获取调度器对象 + :return: + """ + if not self.scheduler: + self.scheduler = AsyncIOScheduler() + return self.scheduler + + def start(self, paused=False): + self.scheduler.start(paused) + + def add_task(self, task: MonitorTask, offset=0, is_random=False): + """ + 添加任务 + :param is_random: 是否随机偏移 + :param offset: 偏移多少秒后执行 + :param task: + :return: + """ + scheduler = self.get_scheduler() + spider: AbstractSpider = None + if task.platform == Platform.XIN_HUA: + spider = XinHuaSpider() + elif task.platform == Platform.REN_MIN: + spider = RenMinSpider() + elif task.platform == Platform.YANG_SHI: + spider = YangShiSpider() + if not spider: + # logger.error(F"未知的平台: {task.platform} 任务id: {task.id}") + return + if not task.gather_time: + logger.error(F"[调度器]采集时间不存在 任务id: {task.id}") + if is_random: + offset = offset + random.randint(1, 29) + # 时间向后偏移 + task_date_time = date_format.gen_job_datetime(task.gather_time) + task_date_time = task_date_time + timedelta(seconds=offset) + + if task_date_time < datetime.datetime.now(): + task_date_time = datetime.datetime.now() + datetime.timedelta(seconds=60) + # 添加定时任务 + scheduler.add_job(spider.start, "date", run_date=task_date_time, kwargs={"task_id": task.id}) + logger.info( + F"[调度器]注册定时任务 ID: {task.id} 执行时间: {task_date_time} {F'偏移{offset}秒后执行' if offset > 0 else ''}") + + def add_tasks(self, tasks: List[MonitorTask], is_random=False): + # 按平台和关键词分组 + group = {} + for task in tasks: + if task.platform not in group: + group[task.platform] = {} + if task.keyword not in group[task.platform]: + group[task.platform][task.keyword] = [] + group[task.platform][task.keyword].append(task) + # 遍历每个关键词组 + for platform, platform_group in group.items(): + for keyword, task_list in platform_group.items(): + sorted_task_list = sorted(task_list, key=lambda e: date_format.parse_time(task.gather_time)) + # 判断最后一个任务是否在极限时间之前 + if date_format.lt_time(sorted_task_list[-1].gather_time, config.MAX_GATHER_TIME): + # 创建一个补偿任务 + new_task = copy.deepcopy(sorted_task_list[-1]) + new_task.gather_time = config.MAX_GATHER_TIME + sorted_task_list.append(new_task) + for sorted_task in sorted_task_list: + self.add_task(sorted_task, 0, is_random) diff --git a/utils/utils.py b/utils/utils.py new file mode 100644 index 0000000..ef4dbcc --- /dev/null +++ b/utils/utils.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import logging +from logging.handlers import TimedRotatingFileHandler +import re +import os + + +def init_loging_config(): + # 检查log文件夹是否存在 + if not os.path.exists("./log"): + os.mkdir("./log") + # 创建一个handler，用于按日期写入日志文件 + # 'W0' 表示每周滚动一次，'D' 表示每天滚动一次，'H' 表示每小时滚动一次，'M' 表示每分钟滚动一次 + # 'midnight' 表示在午夜滚动，'h:m' 表示在指定的小时和分钟滚动 + # backupCount 表示保留的日志文件的个数，超过后会删除最旧的日志文件 + # when='D', interval=1, backupCount=7 表示每天滚动一次，并保留最近7天的日志文件 + file_handler = TimedRotatingFileHandler('./log/huo_spider.log', when='D', interval=1, encoding='utf-8') + file_handler.setLevel(logging.DEBUG) + + # 定义handler的输出格式 + formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s %(message)s ') + file_handler.setFormatter(formatter) + + level = logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s [%(name)s] %(levelname)s %(message)s ", + datefmt='[%Y-%m-%d %H:%M:%S]' + ) + _logger = logging.getLogger("HuoSpider") + _logger.setLevel(level) + _logger.addHandler(file_handler) + return _logger + + +logger = init_loging_config() + + +def is_blank(val: str): + if val is None: + return False + if not val.strip(): + return False + return True + + +def count_characters(val): + """ + 统计中文和非中文字符个数 + :param val: + :return: + """ + if not isinstance(val, str): + val = str(val) + chinese_pattern = re.compile(r'[\u4e00-\u9fa5]') + not_chinese_pattern = re.compile(r'[^\u4e00-\u9fa5]') + chinese = re.findall(chinese_pattern, val) + not_chinese = re.findall(not_chinese_pattern, val) + return len(chinese), len(not_chinese)