From faf0afee8ca3f70ef26b876ed63ca6346bb843ab Mon Sep 17 00:00:00 2001 From: Darkangeel_hd Date: Mon, 4 Jul 2022 03:42:00 +0200 Subject: [PATCH] Updated utils/requestCall Added new options to spoof user agent New Aruppi APi user agent Adapt relevant code to new funcionality fixed line break on urls.ts --- README.md | 2 +- package.json | 2 +- src/controllers/AnimeController.ts | 1 + src/controllers/UtilsController.ts | 3 +- src/utils/animeTheme.ts | 1 + src/utils/requestCall.ts | 52 ++++++++++++++++++++++-------- src/utils/urls.ts | 3 +- src/utils/util.ts | 7 ++-- 8 files changed, 50 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6cfc71e..13c9e0c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# **Aruppi API** (v4.1.8) +# **Aruppi API** (v4.2.0) > This API has everything about Japan, from anime, music, radio, images, videos ... to japanese culture > diff --git a/package.json b/package.json index 4eabd04..3b2435d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aruppi", - "version": "4.1.9", + "version": "4.2.0", "description": "Aruppi is a custom API to obtain data from the Japanese culture for the mobile app", "main": "./src/api/api.ts", "scripts": { diff --git a/src/controllers/AnimeController.ts b/src/controllers/AnimeController.ts index 4292e2c..7e7b76f 100644 --- a/src/controllers/AnimeController.ts +++ b/src/controllers/AnimeController.ts @@ -229,6 +229,7 @@ export default class AnimeController { data = await requestGot(`${urls.BASE_ANIMEFLV}api/animes/list`, { parse: true, scrapy: false, + spoof: true, }); } catch (err) { return next(err); diff --git a/src/controllers/UtilsController.ts b/src/controllers/UtilsController.ts index 3760616..f76917e 100644 --- a/src/controllers/UtilsController.ts +++ b/src/controllers/UtilsController.ts @@ -234,7 +234,7 @@ export default class UtilsController { `${urls.BASE_QWANT}t=images&q=${encodeURIComponent( title, )}&count=51&locale=es_ES&safesearch=1`, - { scrapy: false, parse: true }, + { scrapy: false, parse: true, spoof: true, }, ); } catch (err) { return next(err); @@ -637,6 +637,7 @@ export default class UtilsController { data = await requestGot(`${urls.BASE_THEMEMOE}roulette`, { parse: true, scrapy: false, + spoof: true, }); } catch (err) { return next(err); diff --git a/src/utils/animeTheme.ts b/src/utils/animeTheme.ts index dba47ae..2c06d6e 100644 --- a/src/utils/animeTheme.ts +++ b/src/utils/animeTheme.ts @@ -285,6 +285,7 @@ async function redditocall(href: string) { const resp = await requestGot(urls.REDDIT_ANIMETHEMES + href + '.json', { parse: true, scrapy: false, + spoof: true, }); return cheerio.load(getHTML(resp.data.content_html)); diff --git a/src/utils/requestCall.ts b/src/utils/requestCall.ts index 1f005c4..ad03cbc 100644 --- a/src/utils/requestCall.ts +++ b/src/utils/requestCall.ts @@ -1,28 +1,52 @@ import got from 'got'; import cheerio from 'cheerio'; import { CookieJar } from 'tough-cookie'; +// @ts-ignore +import * as got_pjson from 'got/package.json' +const pjson = require('../../package.json'); const cookieJar = new CookieJar(); +const aruppi_options: any = { + cookieJar, + 'headers': { + 'user-agent': `Aruppi-API/${pjson.version} ${got_pjson.name}/${got_pjson.version}`, + 'x-client': 'aruppi-api' + }, + +}; interface Options { - scrapy: boolean; - parse: boolean; + scrapy?: boolean, + parse?: boolean, + spoof?: boolean } export const requestGot = async ( - url: string, - options?: Options, + url: string, + options?: Options, ): Promise => { - if (options !== undefined) { - if (options.scrapy) { - const response = await got(url, { cookieJar }); - return await cheerio.load(response.body); - } + const got_options: any = {...got.defaults.options, ...aruppi_options} + if (options) { + if (options.spoof != null) { + got_options.headers["user-agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/69.0"; + delete got_options.headers['x-client']; + if (!options.spoof) + got_options.headers['user-agent'] = got.defaults.options.headers['user-agent']; + } else if (process.env.ALPI_KEY && (new URL(url)).hostname.match(/\.jeluchu\.xyz$/)) { + got_options.headers['x-aruppi-key'] = process.env.ALPI_KEY; + } + + if (options.scrapy) { + const response = await got(url, got_options); + return cheerio.load(response.body); + } - if (options.parse) { - return await got(url, { cookieJar }).json(); + if (options.parse) { + got_options.responseType = 'json'; + const response = await got(url, got_options); + return response.body; + } } - } else { - return await got.get(url, { cookieJar }); - } + const response = await got.get(url, got_options); + return response; }; diff --git a/src/utils/urls.ts b/src/utils/urls.ts index a3d2c0d..a439e26 100644 --- a/src/utils/urls.ts +++ b/src/utils/urls.ts @@ -8,8 +8,7 @@ export default { BASE_YOUTUBE: 'https://aruppi.jeluchu.xyz/api/Youtube/?channelId=', BASE_YOUTUBE_PLAYLIST: 'https://aruppi.jeluchu.xyz/api/Youtube/playlist/?playlistId=', BASE_JIKAN: 'https://aruppi.jeluchu.xyz/apis/jikan/v3/', - BASE_IVOOX: - 'https://www.ivoox.com/podcast-anitakume_fg_f1660716_filtro_1.xml', + BASE_IVOOX: 'https://www.ivoox.com/podcast-anitakume_fg_f1660716_filtro_1.xml', BASE_KUDASAI: 'https://somoskudasai.com/feed/', BASE_RAMENPARADOS: 'https://ramenparados.com/category/noticias/anime/feed/', BASE_CRUNCHYROLL: 'https://www.crunchyroll.com/newsrss?lang=esES', diff --git a/src/utils/util.ts b/src/utils/util.ts index 6493258..da74cd6 100644 --- a/src/utils/util.ts +++ b/src/utils/util.ts @@ -280,6 +280,7 @@ export const getRelatedAnimesMAL = async (mal_id: number) => { $ = await requestGot(`https://myanimelist.net/anime/${mal_id}`, { parse: false, scrapy: true, + spoof: true, }); } catch (err) { return err; @@ -606,6 +607,7 @@ export const tioanimeInfo = async (id: string | undefined, mal_id: number) => { $ = await requestGot(`${urls.BASE_TIOANIME}anime/${id}`, { scrapy: true, parse: false, + spoof: true, }); /* Extra info of the anime */ @@ -785,6 +787,7 @@ export const videoServersTioAnime = async (id: string) => { $ = await requestGot(`${urls.BASE_TIOANIME}${id}`, { scrapy: true, parse: false, + spoof: true, }); } catch (err) { return err; @@ -941,7 +944,7 @@ async function desuServerUrl(url: string) { } } - $ = await requestGot(url, {scrapy: true, parse: false}); + $ = await requestGot(url, {scrapy: true, parse: false, spoof: true}); } catch (err) { return err; } @@ -1033,7 +1036,7 @@ export function getThemes(themes: any[]) { } export const imageUrlToBase64 = async (url: string) => { - let img: any = await requestGot(url); + let img: any = await requestGot(url,{spoof:false}); return img.rawBody.toString('base64'); };