Add Directory, fixes and improvements

README.md

@@ -1,4 +1,4 @@
-# **Aruppi API** (v2.2.5)
+# **Aruppi API** (v2.3.0)
 
 > This API has everything about Japan, from anime, music, radio, images, videos ... to japanese culture 
 >

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "aruppi",
-  "version": "2.2.4",
+  "version": "2.3.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aruppi",
-  "version": "2.2.5",
+  "version": "2.3.0",
   "description": "Aruppi is a custom API to obtain data from the Japanese culture for the mobile app",
   "main": "./src/api/api.js",
   "scripts": {
@@ -51,6 +51,7 @@
     "helmet": "^3.22.0",
     "hooman": "^1.2.5",
     "request": "^2.88.0",
-    "rss-to-json": "^1.1.1"
+    "rss-to-json": "^1.1.1",
+    "tough-cookie": "^4.0.0"
   }
 }

src/api/api.js

@@ -1,5 +1,8 @@
 const rss = require('rss-to-json');
-const html = require('got');
+
+const {
+  homgot
+} = require('../api/apiCall');
 
 const {
   animeflvInfo,
@@ -12,6 +15,7 @@ const {
   obtainPreviewNews,
   structureThemes,
   getAnimes,
+  getDirectory,
   helper
 } = require('../utils/index');
 
@@ -21,7 +25,8 @@ const {
 
 const schedule = async (day) =>{
 
-  const data = await html(`${BASE_JIKAN}schedule/${day.current}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_JIKAN}schedule/${day.current}`, options);
   const body = data[day.current];
   const promises = []
 
@@ -40,7 +45,8 @@ const schedule = async (day) =>{
 };
 
 const top = async (type, subtype, page) =>{
-  const data = await html(`${BASE_JIKAN}top/${type}/${page}/${subtype}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_JIKAN}top/${type}/${page}/${subtype}`, options);
   return data.top;
 };
 
@@ -58,6 +64,8 @@ const getAllAnimes = async () =>{
 
 };
 
+const getAllDirectory = async () =>{ return await getDirectory(); };
+
 const getAnitakume = async () =>{
 
   const promises = []
@@ -126,7 +134,8 @@ const getNews = async (pageRss) =>{
 
 const season = async (year, type) =>{
 
-  const data = await html(`${BASE_JIKAN}season/${year}/${type}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_JIKAN}season/${year}/${type}`, options);
   let body = data.anime;
   const promises = []
 
@@ -146,7 +155,8 @@ const season = async (year, type) =>{
 
 const allSeasons = async () =>{
 
-  const data = await html(`${BASE_JIKAN}season/archive`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_JIKAN}season/archive`, options);
   let body = data.archive;
   const promises = []
 
@@ -164,7 +174,8 @@ const allSeasons = async () =>{
 
 const laterSeasons = async () =>{
 
-  const data = await html(`${BASE_JIKAN}season/later`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_JIKAN}season/later`, options);
   let body = data.anime;
   const promises = []
 
@@ -183,7 +194,8 @@ const laterSeasons = async () =>{
 
 const getLastEpisodes = async () =>{
 
-  const data = await html(`${BASE_ANIMEFLV_JELU}LatestEpisodesAdded`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_ANIMEFLV_JELU}LatestEpisodesAdded`, options);
   let body = data.episodes;
   const promises = []
 
@@ -205,7 +217,8 @@ const getLastEpisodes = async () =>{
 
 const getSpecials = async (type, subType, page) =>{
 
-  const data = await html(`${BASE_ANIMEFLV_JELU}${type.url}/${subType}/${page}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_ANIMEFLV_JELU}${type.url}/${subType}/${page}`, options);
   let body = data[type.prop];
   const promises = []
 
@@ -322,7 +335,8 @@ const getMoreInfo = async (title) =>{
 
 const getAnimeServers = async (id) => {
 
-  const data = await html(`${BASE_ANIMEFLV_JELU}GetAnimeServers/${id}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_ANIMEFLV_JELU}GetAnimeServers/${id}`, options);
   let body = data.servers;
 
   return await transformUrlServer(body);
@@ -333,7 +347,8 @@ const search = async (title) =>{ return await searchAnime(title); };
 
 const getImages = async (query) => {
 
-  const data = await html(`${BASE_QWANT}count=${query.count}&q=${query.title}&t=${query.type}&safesearch=${query.safesearch}&locale=${query.country}&uiv=4`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_QWANT}count=${query.count}&q=${query.title}&t=${query.type}&safesearch=${query.safesearch}&locale=${query.country}&uiv=4`, options);
   const body = data.data.result.items;
   const promises = []
 
@@ -353,7 +368,8 @@ const getImages = async (query) => {
 
 const getYoutubeVideos = async (channelId) => {
 
-  const data = await html(`${BASE_YOUTUBE}${channelId.id}&part=${channelId.part}&order=${channelId.order}&maxResults=${channelId.maxResults}`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_YOUTUBE}${channelId.id}&part=${channelId.part}&order=${channelId.order}&maxResults=${channelId.maxResults}`, options);
   const body = data[channelId.prop];
   const promises = []
 
@@ -378,18 +394,40 @@ const getRadioStations = async () => {
 }
 
 const getOpAndEd = async (title) => {
-  const data = await html(`${BASE_THEMEMOE}anime/search/${title}`).json();
+
+  let data
+
+  const special = [
+    { title: 'Kaguya-sama wa Kokurasetai: Tensai-tachi no Renai Zunousen', code: 37999 },
+    { title: 'Kaguya-sama wa Kokurasetai: Tensai-tachi no Renai Zunousen 2nd Season', code: 40591 },
+    { title: 'Princess Connect! Re:Dive', code: 39292 },
+    { title: 'Shachou, Battle no Jikan Desu!', code: 40783 }
+  ];
+
+  for (let name in special) {
+    if (title === special[name].title) {
+      data = JSON.parse("[" + special[name].code + "]")
+      break;
+    }
+  }
+
+  if (data === undefined) {
+    let options = { parse: true }
+    data = await homgot(`${BASE_THEMEMOE}anime/search/${title}`, options);
+  }
+
   return await structureThemes(data, true, 0)
 };
 
 const getThemesSeason = async (year, season) => {
 
   let data
+  let options = { parse: true }
 
   if (season === undefined) {
-    data = await html(`${BASE_THEMEMOE}seasons/${year}`).json();
+    data = await homgot(`${BASE_THEMEMOE}seasons/${year}`, options);
   } else {
-    data = await html(`${BASE_THEMEMOE}seasons/${year}/${season}`).json();
+    data = await homgot(`${BASE_THEMEMOE}seasons/${year}/${season}`, options);
   }
 
   return await structureThemes(data, false, 0)
@@ -397,7 +435,8 @@ const getThemesSeason = async (year, season) => {
 };
 
 const getRandomTheme = async () => {
-  const data = await html(`${BASE_THEMEMOE}roulette`).json();
+  let options = { parse: true }
+  const data = await homgot(`${BASE_THEMEMOE}roulette`, options);
   return await structureThemes(data, true)
 };
 
@@ -405,10 +444,11 @@ const getArtist = async (id) => {
 
   let data
   let promises = []
+  let options = { parse: true }
 
   if (id === undefined) {
 
-    data = await html(`${BASE_THEMEMOE}artists`).json();
+    data = await homgot(`${BASE_THEMEMOE}artists`, options);
     data.map(doc => {
 
       promises.push({
@@ -421,7 +461,7 @@ const getArtist = async (id) => {
     return promises;
 
   } else {
-    data = await html(`${BASE_THEMEMOE}artists/${id}`).json();
+    data = await homgot(`${BASE_THEMEMOE}artists/${id}`, options);
     return await structureThemes(data, false, 1)
   }
 
@@ -431,6 +471,7 @@ module.exports = {
   schedule,
   top,
   getAllAnimes,
+  getAllDirectory,
   getAnitakume,
   getNews,
   season,

src/api/apiCall.js

@@ -0,0 +1,30 @@
+const hooman = require('hooman');
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+const cheerio = require('cheerio');
+
+let response
+let data
+
+const homgot = async (url, options) => {
+
+    response = await hooman.get(url, cookieJar);
+
+    if (options !== undefined) {
+        if (options.scrapy) {
+            data = await cheerio.load(response.body)
+        }
+        if (options.parse) {
+            data = JSON.parse(response.body)
+        }
+    } else {
+        data = response
+    }
+
+    return data
+
+}
+
+module.exports = {
+    homgot
+}

src/api/index.js

@@ -7,7 +7,7 @@ router.get('/', (req, res) => {
   res.json({
     message: 'Aruppi API - 🎏',
     author: 'Jéluchu',
-    version: '2.2.5',
+    version: '2.3.0',
     credits: 'The bitch loves APIs that offers data to Aruppi App',
     entries: [
       {
@@ -18,6 +18,7 @@ router.get('/', (req, res) => {
         'News': '/api/v2/news',
         'Season': '/api/v2/season/:year/:type',
         'All Seasons': '/api/v2/allSeasons',
+        'All Directory': '/api/v2/allDirectory',
         'Futures Seasons': '/api/v2/laterSeasons',
         'LastEpisodes': '/api/v2/lastEpisodes',
         'Movies': '/api/v2/movies/:type/:page',

src/api/routes/index.js

@@ -61,6 +61,23 @@ router.get('/allAnimes' , (req, res) =>{
 
 });
 
+router.get('/allDirectory' , (req, res) =>{
+
+    api.getAllDirectory()
+        .then(directory =>{
+            if (directory.length > 0) {
+                res.status(200).json({
+                    directory
+                });
+            } else (
+                res.status(500).json({ message: 'Aruppi lost in the shell'})
+            )
+        }).catch((err) =>{
+        console.error(err);
+    });
+
+});
+
 router.get('/anitakume' , (req, res) =>{
 
     api.getAnitakume()

src/api/urls.js

@@ -10,6 +10,7 @@ module.exports = {
   BASE_RAMENPARADOS: 'https://ramenparados.com/category/noticias/anime/feed/',
   BASE_CRUNCHYROLL: 'https://www.crunchyroll.com/newsrss?lang=esES',
   SEARCH_URL: 'https://animeflv.net/browse?q=',
+  SEARCH_DIRECTORY: 'https://animeflv.net/browse?order=title&page=',
   BASE_EPISODE_IMG_URL: 'https://cdn.animeflv.net/screenshots/',
   BASE_QWANT: 'https://api.qwant.com/search/images?',
   BASE_THEMEMOE: 'https://themes.moe/api/'

src/utils/index.js

@@ -1,11 +1,11 @@
-const cheerio = require('cheerio');
-const html = require('got');
-const hooman = require('hooman');
-
 const {
-    BASE_ANIMEFLV, BASE_JIKAN, BASE_EPISODE_IMG_URL, SEARCH_URL, BASE_ARUPPI, BASE_THEMEMOE
+    BASE_ANIMEFLV, BASE_JIKAN, BASE_EPISODE_IMG_URL, SEARCH_URL, SEARCH_DIRECTORY, BASE_ARUPPI, BASE_THEMEMOE
 } = require('../api/urls');
 
+const {
+    homgot
+} = require('../api/apiCall');
+
 const animeflvInfo = async (id, index) => {
 
     let poster = ""
@@ -14,19 +14,12 @@ const animeflvInfo = async (id, index) => {
     let rating = ""
     let debut = ""
     let type = ""
-
-    let res
     let $
 
     try {
 
-        try {
-            res = await html(`${BASE_ANIMEFLV}anime/${id}`);
-            $ = await cheerio.load(res.body);
-        } catch (error) {
-            res = await hooman.get(`${BASE_ANIMEFLV}anime/${id}`);
-            $ = await cheerio.load(res.body)
-        }
+        let options = { scrapy: true }
+        $ = await homgot(`${BASE_ANIMEFLV}anime/${id}`, options);
 
         const scripts = $('script');
         const anime_info_ids = [];
@@ -116,13 +109,16 @@ const animeflvInfo = async (id, index) => {
 };
 
 const getAnimeCharacters = async (title) => {
-    const res = await html(`${BASE_JIKAN}search/anime?q=${title}`).json();
+
+    let options = { parse: true }
+
+    const res = await homgot(`${BASE_JIKAN}search/anime?q=${title}`, options);
     const matchAnime = res.results.filter(x => x.title === title);
     const malId = matchAnime[0].mal_id;
 
     if (typeof matchAnime[0].mal_id === 'undefined') return null;
 
-    const data = await html(`${BASE_JIKAN}anime/${malId}/characters_staff`).json();
+    const data = await homgot(`${BASE_JIKAN}anime/${malId}/characters_staff`, options);
     let body = data.characters;
 
     if (typeof body === 'undefined') return null;
@@ -153,13 +149,15 @@ const getAnimeCharacters = async (title) => {
 };
 
 const getAnimeVideoPromo = async (title) => {
-    const res = await html(`${BASE_JIKAN}search/anime?q=${title}`).json();
+
+    let options = { parse: true }
+    const res = await homgot(`${BASE_JIKAN}search/anime?q=${title}`, options);
     const matchAnime = res.results.filter(x => x.title === title);
     const malId = matchAnime[0].mal_id;
 
     if (typeof matchAnime[0].mal_id === 'undefined') return null;
 
-    const data = await html(`${BASE_JIKAN}anime/${malId}/videos`).json();
+    const data = await homgot(`${BASE_JIKAN}anime/${malId}/videos`, options);
     const body = data.promo;
     const promises = [];
 
@@ -175,13 +173,15 @@ const getAnimeVideoPromo = async (title) => {
 };
 
 const animeExtraInfo = async (title) => {
-    const res = await html(`${BASE_JIKAN}search/anime?q=${title}`).json();
+
+    let options = { parse: true }
+    const res = await homgot(`${BASE_JIKAN}search/anime?q=${title}`, options);
     const matchAnime = res.results.filter(x => x.title === title);
     const malId = matchAnime[0].mal_id;
 
     if (typeof matchAnime[0].mal_id === 'undefined') return null;
 
-    const data = await html(`${BASE_JIKAN}anime/${malId}`).json();
+    const data = await homgot(`${BASE_JIKAN}anime/${malId}`, options);
     const body = Array(data);
     const promises = [];
 
@@ -227,7 +227,7 @@ const animeExtraInfo = async (title) => {
 };
 
 const imageUrlToBase64 = async (url) => {
-    let img = await hooman.get(url)
+    let img = await homgot(url)
     return img.rawBody.toString('base64');
 };
 
@@ -235,18 +235,11 @@ const helper = async () => {}
 
 const searchAnime = async (query) => {
 
-    let res
     let $
     let promises = []
 
-    try {
-        res = await html(`${SEARCH_URL}${query}`);
-        $ = await cheerio.load(res.body);
-    } catch (error) {
-        res = await hooman.get(`${SEARCH_URL}${query}`);
-        $ = await cheerio.load(res.body)
-    }
-
+    let options = { scrapy: true }
+    $ = await homgot(`${SEARCH_URL}${query}`, options);
     $('div.Container ul.ListAnimes li article').each((index, element) => {
         const $element = $(element);
         const id = $element.find('div.Description a.Button').attr('href').slice(1);
@@ -275,12 +268,10 @@ const transformUrlServer = async (urlReal) => {
 
     for (const index in urlReal) {
         if (urlReal[index].server === 'amus' || urlReal[index].server === 'natsuki') {
-            try {
-                res = await html(urlReal[index].code.replace("embed", "check")).json();
-            } catch (e) {
-                data = await hooman.get(urlReal[index].code.replace("embed", "check"));
-                res = JSON.parse(data.body)
-            }
+
+            let options = { parse: true }
+            res = await homgot(urlReal[index].code.replace("embed", "check"), options);
+
             urlReal[index].code = res.file || null
             urlReal[index].direct = true
         } else {
@@ -338,7 +329,8 @@ const structureThemes = async (body, indv, task) => {
         for (let i = 0; i <= body.length - 1; i++) {
 
             if (indv === true) {
-                data = await html(`${BASE_THEMEMOE}themes/${body[i]}`).json();
+                let options = { parse: true }
+                data = await homgot(`${BASE_THEMEMOE}themes/${body[i]}`, options);
                 themes = await getThemes(data[0].themes)
             } else {
                 data = body
@@ -427,19 +419,44 @@ const getThemes = async (themes) => {
 };
 
 const getAnimes = async () => {
+    let options = { parse: true }
+    return await homgot(`${BASE_ANIMEFLV}api/animes/list`, options);
+};
 
-    let res
-    let data
 
-    try {
-        data = await html(`${BASE_ANIMEFLV}api/animes/list`).json();
-    } catch (error) {
-        res = await hooman.get(`${BASE_ANIMEFLV}api/animes/list`)
-        data = JSON.parse(res.body);
-    }
+const getDirectory = async () => {
+
+    let $
+    let promises = []
+
+    let options = { scrapy: true }
+    $ = await homgot(`${SEARCH_URL}`, options);
+    const lastPage = $('body div.Wrapper div.Container main div.NvCnAnm ul li a')[11].children[0].data
 
-    return data;
+    for (let i = 1; i <= lastPage; i++) {
 
+        let options = { scrapy: true }
+        $ = await homgot(`${SEARCH_DIRECTORY}${i}`, options);
+
+        $('div.Container ul.ListAnimes li article').each((index, element) => {
+            const $element = $(element);
+            const id = $element.find('div.Description a.Button').attr('href').slice(1);
+            const title = $element.find('a h3').text();
+            let poster = $element.find('a div.Image figure img').attr('src') || $element.find('a div.Image figure img').attr('data-cfsrc');
+            const type = $element.find('div.Description p span.Type').text();
+
+            promises.push(helper().then(async () => ({
+                id: id || null,
+                title: title || null,
+                type: type || null,
+                image: await imageUrlToBase64(poster) || null
+            })));
+
+        })
+
+    }
+
+    return Promise.all(promises);
 };
 
 module.exports = {
@@ -454,5 +471,6 @@ module.exports = {
     structureThemes,
     getThemes,
     getAnimes,
+    getDirectory,
     helper
 }