🧱 To work the themeParser and adding more routes

.env.example

@@ -1,2 +1,5 @@
-PORT=
+PORT_LISTEN=5000
-DATABASE=
+DATABASE_HOST=localhost
+DATABASE_PORT=27017
+REDIS_HOST=localhost
+REDIS_PORT=6379

package.json

@@ -54,6 +54,7 @@
     "helmet": "^4.4.1",
     "mongodb": "^3.6.4",
     "mongoose": "^5.11.18",
+    "redis": "^3.0.2",
     "rss-parser": "^3.12.0",
     "tough-cookie": "^4.0.0",
     "ts-node-dev": "^1.1.1"
@@ -63,6 +64,7 @@
     "@types/cors": "^2.8.10",
     "@types/express": "^4.17.11",
     "@types/node": "^14.14.31",
+    "@types/redis": "^2.8.28",
     "@types/tough-cookie": "^4.0.0",
     "@typescript-eslint/eslint-plugin": "^4.15.2",
     "@typescript-eslint/parser": "^4.15.2",

src/controllers/AnimeController.ts

@@ -271,4 +271,8 @@ export default class AnimeController {
       return next(err);
     }
   }
+
+  async getAnimeGenres(req: Request, res: Response, next: NextFunction) {
+    const { genre, order, page } = req.params;
+  }
 }

src/controllers/UtilsController.ts

@@ -6,6 +6,10 @@ import { requestGot } from '../utils/requestCall';
 import RadioStationModel, {
   RadioStation,
 } from '../database/models/radiostation.model';
+import ThemeModel, { Theme } from '../database/models/theme.model';
+import ThemeParser from '../utils/animeTheme';
+import { structureThemes } from '../utils/util';
+import { getThemes } from '../utils/util';
 
 /*
   UtilsController - controller to parse the
@@ -13,6 +17,8 @@ import RadioStationModel, {
   parsing RSS.
 */
 
+const themeParser = new ThemeParser();
+
 type CustomFeed = {
   foo: string;
 };
@@ -303,4 +309,110 @@ export default class UtilsController {
       res.status(500).json({ message: 'Aruppi lost in the shell' });
     }
   }
+
+  async getAllThemes(req: Request, res: Response, next: NextFunction) {
+    let data: Theme[];
+
+    try {
+      data = await ThemeModel.find();
+    } catch (err) {
+      return next(err);
+    }
+
+    const results: any[] = data.map((item: Theme) => {
+      return {
+        id: item.id,
+        title: item.title,
+        year: item.year,
+        themes: item.themes,
+      };
+    });
+
+    if (results.length > 0) {
+      res.status(200).json({ themes: results });
+    } else {
+      res.status(500).json({ message: 'Aruppi lost in the shell' });
+    }
+  }
+
+  async getOpAndEd(req: Request, res: Response, next: NextFunction) {
+    const { title } = req.params;
+    let result: any;
+
+    try {
+      result = await structureThemes(await themeParser.serie(title), true);
+    } catch (err) {
+      return next(err);
+    }
+
+    if (result) {
+      res.status(200).json({ result });
+    } else {
+      res.status(500).json({ message: 'Aruppi lost in the shell' });
+    }
+  }
+
+  async getThemesYear(req: Request, res: Response, next: NextFunction) {
+    const { year } = req.params;
+    let data: any;
+
+    try {
+      if (year === undefined) {
+        data = await themeParser.allYears();
+      } else {
+        data = await structureThemes(await themeParser.year(year), false);
+      }
+    } catch (err) {
+      return next(err);
+    }
+
+    if (data.length > 0) {
+      res.status(200).json({ data });
+    } else {
+      res.status(500).json({ message: 'Aruppi lost in the shell' });
+    }
+  }
+
+  async randomTheme(req: Request, res: Response, next: NextFunction) {
+    let data: any;
+
+    try {
+      data = await requestGot(`${urls.BASE_THEMEMOE}roulette`, {
+        parse: true,
+        scrapy: false,
+      });
+    } catch (err) {
+      return next(err);
+    }
+
+    const result: any[] = getThemes(data.themes);
+
+    if (result.length > 0) {
+      res.set('Cache-Control', 'no-store');
+      res.status(200).json({ result });
+    } else {
+      res.status(500).json({ message: 'Aruppi lost in the shell' });
+    }
+  }
+
+  async getArtist(req: Request, res: Response, next: NextFunction) {
+    const { id } = req.params;
+    let data: any;
+
+    try {
+      if (id === undefined) {
+        data = await themeParser.artists();
+      } else {
+        data = await structureThemes(await themeParser.artist(id), false);
+      }
+    } catch (err) {
+      return next(err);
+    }
+
+    if (data.length > 0) {
+      res.status(200).json({ data });
+    } else {
+      res.status(500).json({ message: 'Aruppi lost in the shell' });
+    }
+  }
 }

src/database/connection.ts

@@ -1,15 +1,22 @@
 import mongoose from 'mongoose';
+// import redis, { RedisClient } from 'redis';
 
 /* 
   Create the connection to the database
   of mongodb.
 */
 
-export const createConnection = (database: string | undefined) => {
+export const createConnectionMongo = (databaseObj: {
-  mongoose.connect(`mongodb://${database}/anime-directory`, {
+  port: string | undefined;
-    useNewUrlParser: true,
+  host: string | undefined;
-    useUnifiedTopology: true,
+}) => {
-  });
+  mongoose.connect(
+    `mongodb://${databaseObj.host}:${databaseObj.port}/anime-directory`,
+    {
+      useNewUrlParser: true,
+      useUnifiedTopology: true,
+    },
+  );
 
   mongoose.connection.on('error', err => {
     console.log('err', err);
@@ -18,3 +25,17 @@ export const createConnection = (database: string | undefined) => {
     console.log('Database connected: mongoose.');
   });
 };
+
+// export const createConnectionRedis = (redisObj: {
+//   host: string;
+//   port: number;
+// }) => {
+//   const client: RedisClient = redis.createClient({
+//     host: redisObj.host,
+//     port: redisObj.port,
+//   });
+
+//   client.on('connect', () => {
+//     console.log('Redis connected: redis.');
+//   });
+// };

src/routes.ts

@@ -98,5 +98,10 @@ routes.get('/api/v4/images/:title', utilsController.getImages);
 routes.get('/api/v4/videos/:channelId', utilsController.getVideos);
 routes.get('/api/v4/sectionedVideos/:type', utilsController.getSectionVideos);
 routes.get('/api/v4/radio', utilsController.getRadioStations);
+routes.get('/api/v4/allThemes', utilsController.getAllThemes);
+routes.get('/api/v4/themes/:title', utilsController.getOpAndEd);
+routes.get('/api/v4/themesYear/:year?', utilsController.getThemesYear);
+routes.get('/api/v4/randomTheme', utilsController.randomTheme);
+routes.get('/api/v4/artists/:id?', utilsController.getArtist);
 
 export default routes;

src/server.ts

@@ -3,13 +3,23 @@ import cors from 'cors';
 import helmet from 'helmet';
 import dotenv from 'dotenv';
 import { errorHandler, notFound } from './middlewares/middleware';
-import { createConnection } from './database/connection';
+import {
+  createConnectionMongo,
+  // createConnectionRedis,
+} from './database/connection';
 import routes from './routes';
 
 const app: Application = express();
 
 dotenv.config();
-createConnection(process.env.DATABASE);
+createConnectionMongo({
+  host: process.env.DATABASE_HOST,
+  port: process.env.DATABASE_PORT,
+});
+// createConnectionRedis({
+//   host: process.env.REDIS_HOST!,
+//   port: parseInt(process.env.REDIS_PORT!),
+// });
 app.use(cors());
 app.use(helmet());
 app.use(express.json());
@@ -24,4 +34,4 @@ app.use(errorHandler);
   is going to listen in the server.
   ex: PORT=3000.
 */
-app.listen(process.env.PORT || 3000);
+app.listen(process.env.PORT_LISTEN || 3000);

src/utils/animeTheme.ts

@@ -0,0 +1,301 @@
+import cheerio from 'cheerio';
+import { requestGot } from './requestCall';
+import urls from './urls';
+
+export default class ThemeParser {
+  animes: any[] = [];
+  $: any = '';
+
+  async all() {
+    try {
+      this.animes = [];
+      this.$ = await redditocall('year_index');
+      return await this.parseLinks();
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async allYears() {
+    try {
+      this.animes = [];
+      this.$ = await redditocall('year_index');
+      return await this.parseYears();
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async serie(title: string) {
+    try {
+      this.animes = [];
+      this.$ = await redditocall('anime_index');
+      return await this.parseSerie(title);
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async artists() {
+    try {
+      this.animes = [];
+      this.$ = await redditocall('artist');
+      return await this.parseArtists();
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async artist(id: string) {
+    try {
+      this.animes = [];
+      this.$ = await redditocall(`artist/${id}`);
+      return await this.parseArtist();
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async random() {
+    try {
+      this.animes = [];
+      this.$ = await redditocall('anime_index');
+      return await this.parseRandom();
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  async year(date: string) {
+    let animes: any = [];
+
+    this.$ = await redditocall(date);
+    this.$('h3').each((index: number, element: cheerio.Element) => {
+      let parsed = this.parseAnime(this.$(element));
+      parsed.year = date;
+      animes.push(parsed);
+    });
+    return animes;
+  }
+
+  parseRandom() {
+    return new Promise(async resolve => {
+      let data = this.$('p a');
+      const origin: any = '1';
+      let randomize = Math.round(
+        Math.random() * (data.length - 1 - origin) + parseInt(origin),
+      );
+
+      this.$ = await redditocall(
+        this.$('p a')
+          [randomize].attribs.href.split('/r/AnimeThemes/wiki/')[1]
+          .split('#wiki')[0],
+      );
+
+      let rand = Math.round(Math.random() * this.$('h3').length - 1);
+      let parsed = this.parseAnime(this.$('h3')[rand]);
+      resolve(parsed);
+    });
+  }
+
+  /*  -ParseYears
+      Get the data from the year
+      get the name and the id to do the respective
+      scrapping.
+  */
+  parseYears() {
+    return new Promise(async resolve => {
+      let years: any[] = [];
+
+      this.$('h3 a').each((index: number, element: cheerio.Element) => {
+        years.push({
+          id: this.$(element).attr('href').split('/')[4],
+          name: this.$(element).text(),
+        });
+      });
+
+      resolve(years);
+    });
+  }
+
+  parseArtists() {
+    return new Promise(async resolve => {
+      let promises = [];
+      let data = this.$('p a').filter((x: any) => x > 0);
+
+      for (let i = 0; i < data.length; i++) {
+        promises.push({
+          id: data[i].children[0].parent.attribs.href.split('/')[5],
+          name: data[i].children[0].data,
+        });
+
+        if (i === data.length - 1) {
+          resolve(promises);
+        }
+      }
+    });
+  }
+
+  parseArtist() {
+    return new Promise(async resolve => {
+      let promises = [];
+      let data = this.$('h3');
+
+      for (let i = 0; i < data.length; i++) {
+        let parsed = await this.parseAnime(data[i]);
+        promises.push(parsed);
+
+        if (i === data.length - 1) {
+          resolve(promises);
+        }
+      }
+    });
+  }
+
+  /*  - ParseSerie
+      Parse the HTML from the redditocall
+      and search for the h3 tag to be the 
+      same of the title and resolve a object.
+  */
+  parseSerie(title: string) {
+    return new Promise(async resolve => {
+      let data = this.$('p a');
+
+      for (let i = 0; i < data.length; i++) {
+        let serieElement = data[i].children[0].data;
+
+        if (serieElement.split(' (')[0] === title) {
+          let year = this.$('p a')
+            [i].attribs.href.split('/r/AnimeThemes/wiki/')[1]
+            .split('#wiki')[0];
+          this.$ = await redditocall(
+            this.$('p a')
+              [i].attribs.href.split('/r/AnimeThemes/wiki/')[1]
+              .split('#wiki')[0],
+          );
+
+          for (let i = 0; i < this.$('h3').length; i++) {
+            if (this.$('h3')[i].children[0].children[0].data === title) {
+              let parsed = this.parseAnime(this.$('h3')[i]);
+              parsed.year = year;
+              resolve(parsed);
+            }
+          }
+        }
+      }
+    });
+  }
+
+  parseLinks() {
+    return new Promise(async resolve => {
+      let years = this.$('h3 a');
+
+      for (let i = 0; i < years.length; i++) {
+        let yearElement = years[i];
+
+        await this.year(this.$(yearElement).attr('href').split('/')[4]).then(
+          async animes => {
+            this.animes = this.animes.concat(animes);
+            if (i === years.length - 1) {
+              resolve(this.animes);
+            }
+          },
+        );
+      }
+    });
+  }
+
+  /* - ParseAnime
+      Parse the h3 tag and get the table
+      for the next function to parse the table
+      and get the information about the ending and
+      openings.
+  */
+  parseAnime(element: cheerio.Element) {
+    let el = this.$(element).find('a');
+    let title = this.$(el).text();
+    let mal_id = this.$(el).attr('href').split('/')[4];
+    let next = this.$(element).next();
+
+    let theme: any = {
+      id: mal_id,
+      title,
+    };
+
+    if (this.$(next).prop('tagName') === 'TABLE') {
+      theme.themes = this.parseTable(this.$(next));
+    } else if (this.$(next).prop('tagName') === 'P') {
+      theme.themes = this.parseTable(this.$(next).next());
+    }
+
+    return theme;
+  }
+
+  /*  - ParseTable
+      Parse the table tag from the HTML
+      and returns a object with all the
+      information.
+  */
+  parseTable(element: cheerio.Element): any {
+    if (this.$(element).prop('tagName') !== 'TABLE') {
+      return this.parseTable(this.$(element).next());
+    }
+
+    let themes: any = [];
+
+    this.$(element)
+      .find('tbody')
+      .find('tr')
+      .each((index: number, element: cheerio.Element) => {
+        let name = replaceAll(
+          this.$(element).find('td').eq(0).text(),
+          '&quot;',
+          '"',
+        );
+        let link = this.$(element).find('td').eq(1).find('a').attr('href');
+        let linkDesc = this.$(element).find('td').eq(1).find('a').text();
+        let episodes =
+          this.$(element).find('td').eq(2).text().length > 0
+            ? this.$(element).find('td').eq(2).text()
+            : '';
+        let notes =
+          this.$(element).find('td').eq(3).text().length > 0
+            ? this.$(element).find('td').eq(3).text()
+            : '';
+
+        themes.push({
+          name,
+          link,
+          desc: linkDesc,
+          type: name.startsWith('OP')
+            ? 'Opening'
+            : name.startsWith('ED')
+            ? 'Ending'
+            : 'OP/ED',
+          episodes,
+          notes,
+        });
+      });
+
+    return themes;
+  }
+}
+
+async function redditocall(href: string) {
+  const resp = await requestGot(urls.REDDIT_ANIMETHEMES + href + '.json', {
+    parse: true,
+    scrapy: false,
+  });
+
+  return cheerio.load(getHTML(resp.data.content_html));
+}
+
+function getHTML(str: string) {
+  let html = replaceAll(str, '&lt;', '<');
+  html = replaceAll(html, '&gt;', '>');
+  return html;
+}
+
+function replaceAll(str: string, find: string, replace: string) {
+  return str.replace(new RegExp(find, 'g'), replace);
+}

src/utils/util.ts

@@ -409,3 +409,49 @@ async function desuServerUrl(url: string) {
 
   return result;
 }
+
+export const structureThemes = async (body: any, indv: boolean) => {
+  let themes: any[] = [];
+
+  if (indv === true) {
+    return {
+      title: body.title,
+      year: body.year,
+      themes: await getThemesData(body.themes),
+    };
+  } else {
+    for (let i = 0; i <= body.length - 1; i++) {
+      themes.push({
+        title: body[i].title,
+        year: body[i].year,
+        themes: await getThemesData(body[i].themes),
+      });
+    }
+
+    return themes;
+  }
+};
+
+function getThemesData(themes: any[]): any {
+  let items: any[] = [];
+
+  for (let i = 0; i <= themes.length - 1; i++) {
+    items.push({
+      title: themes[i].name.split('"')[1] || 'Remasterización',
+      type: themes[i].type,
+      episodes: themes[i].episodes !== '' ? themes[i].episodes : null,
+      notes: themes[i].notes !== '' ? themes[i].notes : null,
+      video: themes[i].link,
+    });
+  }
+
+  return items.filter(x => x.title !== 'Remasterización');
+}
+
+export function getThemes(themes: any[]) {
+  return themes.map((item: any) => ({
+    name: item.themeName,
+    type: item.themeType,
+    video: item.mirror.mirrorURL,
+  }));
+}

yarn.lock

@@ -174,6 +174,13 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c"
   integrity sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==
 
+"@types/redis@^2.8.28":
+  version "2.8.28"
+  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.28.tgz#5862b2b64aa7f7cbc76dafd7e6f06992b52c55e3"
+  integrity sha512-8l2gr2OQ969ypa7hFOeKqtFoY70XkHxISV0pAwmQ2nm6CSPb1brmTmqJCGGrekCo+pAZyWlNXr+Kvo6L/1wijA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/responselike@*", "@types/responselike@^1.0.0":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29"
@@ -2317,6 +2324,33 @@ redent@^1.0.0:
     indent-string "^2.1.0"
     strip-indent "^1.0.1"
 
+redis-commands@^1.5.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
+  integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
+
+redis-errors@^1.0.0, redis-errors@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+  integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
+
+redis-parser@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+  integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
+  dependencies:
+    redis-errors "^1.0.0"
+
+redis@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a"
+  integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==
+  dependencies:
+    denque "^1.4.1"
+    redis-commands "^1.5.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+
 regexp-clone@1.0.0, regexp-clone@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/regexp-clone/-/regexp-clone-1.0.0.tgz#222db967623277056260b992626354a04ce9bf63"