diff --git a/.env.example b/.env.example index 772484b..148a057 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,5 @@ -PORT= -DATABASE= \ No newline at end of file +PORT_LISTEN=5000 +DATABASE_HOST=localhost +DATABASE_PORT=27017 +REDIS_HOST=localhost +REDIS_PORT=6379 \ No newline at end of file diff --git a/package.json b/package.json index 2bcc140..ccd5474 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "helmet": "^4.4.1", "mongodb": "^3.6.4", "mongoose": "^5.11.18", + "redis": "^3.0.2", "rss-parser": "^3.12.0", "tough-cookie": "^4.0.0", "ts-node-dev": "^1.1.1" @@ -63,6 +64,7 @@ "@types/cors": "^2.8.10", "@types/express": "^4.17.11", "@types/node": "^14.14.31", + "@types/redis": "^2.8.28", "@types/tough-cookie": "^4.0.0", "@typescript-eslint/eslint-plugin": "^4.15.2", "@typescript-eslint/parser": "^4.15.2", diff --git a/src/controllers/AnimeController.ts b/src/controllers/AnimeController.ts index 6cf2390..895931a 100644 --- a/src/controllers/AnimeController.ts +++ b/src/controllers/AnimeController.ts @@ -271,4 +271,8 @@ export default class AnimeController { return next(err); } } + + async getAnimeGenres(req: Request, res: Response, next: NextFunction) { + const { genre, order, page } = req.params; + } } diff --git a/src/controllers/UtilsController.ts b/src/controllers/UtilsController.ts index 9622044..e92fd78 100644 --- a/src/controllers/UtilsController.ts +++ b/src/controllers/UtilsController.ts @@ -6,6 +6,10 @@ import { requestGot } from '../utils/requestCall'; import RadioStationModel, { RadioStation, } from '../database/models/radiostation.model'; +import ThemeModel, { Theme } from '../database/models/theme.model'; +import ThemeParser from '../utils/animeTheme'; +import { structureThemes } from '../utils/util'; +import { getThemes } from '../utils/util'; /* UtilsController - controller to parse the @@ -13,6 +17,8 @@ import RadioStationModel, { parsing RSS. */ +const themeParser = new ThemeParser(); + type CustomFeed = { foo: string; }; @@ -303,4 +309,110 @@ export default class UtilsController { res.status(500).json({ message: 'Aruppi lost in the shell' }); } } + + async getAllThemes(req: Request, res: Response, next: NextFunction) { + let data: Theme[]; + + try { + data = await ThemeModel.find(); + } catch (err) { + return next(err); + } + + const results: any[] = data.map((item: Theme) => { + return { + id: item.id, + title: item.title, + year: item.year, + themes: item.themes, + }; + }); + + if (results.length > 0) { + res.status(200).json({ themes: results }); + } else { + res.status(500).json({ message: 'Aruppi lost in the shell' }); + } + } + + async getOpAndEd(req: Request, res: Response, next: NextFunction) { + const { title } = req.params; + let result: any; + + try { + result = await structureThemes(await themeParser.serie(title), true); + } catch (err) { + return next(err); + } + + if (result) { + res.status(200).json({ result }); + } else { + res.status(500).json({ message: 'Aruppi lost in the shell' }); + } + } + + async getThemesYear(req: Request, res: Response, next: NextFunction) { + const { year } = req.params; + let data: any; + + try { + if (year === undefined) { + data = await themeParser.allYears(); + } else { + data = await structureThemes(await themeParser.year(year), false); + } + } catch (err) { + return next(err); + } + + if (data.length > 0) { + res.status(200).json({ data }); + } else { + res.status(500).json({ message: 'Aruppi lost in the shell' }); + } + } + + async randomTheme(req: Request, res: Response, next: NextFunction) { + let data: any; + + try { + data = await requestGot(`${urls.BASE_THEMEMOE}roulette`, { + parse: true, + scrapy: false, + }); + } catch (err) { + return next(err); + } + + const result: any[] = getThemes(data.themes); + + if (result.length > 0) { + res.set('Cache-Control', 'no-store'); + res.status(200).json({ result }); + } else { + res.status(500).json({ message: 'Aruppi lost in the shell' }); + } + } + + async getArtist(req: Request, res: Response, next: NextFunction) { + const { id } = req.params; + let data: any; + + try { + if (id === undefined) { + data = await themeParser.artists(); + } else { + data = await structureThemes(await themeParser.artist(id), false); + } + } catch (err) { + return next(err); + } + + if (data.length > 0) { + res.status(200).json({ data }); + } else { + res.status(500).json({ message: 'Aruppi lost in the shell' }); + } + } } diff --git a/src/database/connection.ts b/src/database/connection.ts index 3a257cb..754a038 100644 --- a/src/database/connection.ts +++ b/src/database/connection.ts @@ -1,15 +1,22 @@ import mongoose from 'mongoose'; +// import redis, { RedisClient } from 'redis'; /* Create the connection to the database of mongodb. */ -export const createConnection = (database: string | undefined) => { - mongoose.connect(`mongodb://${database}/anime-directory`, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); +export const createConnectionMongo = (databaseObj: { + port: string | undefined; + host: string | undefined; +}) => { + mongoose.connect( + `mongodb://${databaseObj.host}:${databaseObj.port}/anime-directory`, + { + useNewUrlParser: true, + useUnifiedTopology: true, + }, + ); mongoose.connection.on('error', err => { console.log('err', err); @@ -18,3 +25,17 @@ export const createConnection = (database: string | undefined) => { console.log('Database connected: mongoose.'); }); }; + +// export const createConnectionRedis = (redisObj: { +// host: string; +// port: number; +// }) => { +// const client: RedisClient = redis.createClient({ +// host: redisObj.host, +// port: redisObj.port, +// }); + +// client.on('connect', () => { +// console.log('Redis connected: redis.'); +// }); +// }; diff --git a/src/routes.ts b/src/routes.ts index de650af..9f22ad8 100644 --- a/src/routes.ts +++ b/src/routes.ts @@ -98,5 +98,10 @@ routes.get('/api/v4/images/:title', utilsController.getImages); routes.get('/api/v4/videos/:channelId', utilsController.getVideos); routes.get('/api/v4/sectionedVideos/:type', utilsController.getSectionVideos); routes.get('/api/v4/radio', utilsController.getRadioStations); +routes.get('/api/v4/allThemes', utilsController.getAllThemes); +routes.get('/api/v4/themes/:title', utilsController.getOpAndEd); +routes.get('/api/v4/themesYear/:year?', utilsController.getThemesYear); +routes.get('/api/v4/randomTheme', utilsController.randomTheme); +routes.get('/api/v4/artists/:id?', utilsController.getArtist); export default routes; diff --git a/src/server.ts b/src/server.ts index 472d20d..ffb21af 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,13 +3,23 @@ import cors from 'cors'; import helmet from 'helmet'; import dotenv from 'dotenv'; import { errorHandler, notFound } from './middlewares/middleware'; -import { createConnection } from './database/connection'; +import { + createConnectionMongo, + // createConnectionRedis, +} from './database/connection'; import routes from './routes'; const app: Application = express(); dotenv.config(); -createConnection(process.env.DATABASE); +createConnectionMongo({ + host: process.env.DATABASE_HOST, + port: process.env.DATABASE_PORT, +}); +// createConnectionRedis({ +// host: process.env.REDIS_HOST!, +// port: parseInt(process.env.REDIS_PORT!), +// }); app.use(cors()); app.use(helmet()); app.use(express.json()); @@ -24,4 +34,4 @@ app.use(errorHandler); is going to listen in the server. ex: PORT=3000. */ -app.listen(process.env.PORT || 3000); +app.listen(process.env.PORT_LISTEN || 3000); diff --git a/src/utils/animeTheme.ts b/src/utils/animeTheme.ts new file mode 100644 index 0000000..e885f1a --- /dev/null +++ b/src/utils/animeTheme.ts @@ -0,0 +1,301 @@ +import cheerio from 'cheerio'; +import { requestGot } from './requestCall'; +import urls from './urls'; + +export default class ThemeParser { + animes: any[] = []; + $: any = ''; + + async all() { + try { + this.animes = []; + this.$ = await redditocall('year_index'); + return await this.parseLinks(); + } catch (err) { + console.log(err); + } + } + + async allYears() { + try { + this.animes = []; + this.$ = await redditocall('year_index'); + return await this.parseYears(); + } catch (err) { + console.log(err); + } + } + + async serie(title: string) { + try { + this.animes = []; + this.$ = await redditocall('anime_index'); + return await this.parseSerie(title); + } catch (err) { + console.log(err); + } + } + + async artists() { + try { + this.animes = []; + this.$ = await redditocall('artist'); + return await this.parseArtists(); + } catch (err) { + console.log(err); + } + } + + async artist(id: string) { + try { + this.animes = []; + this.$ = await redditocall(`artist/${id}`); + return await this.parseArtist(); + } catch (err) { + console.log(err); + } + } + + async random() { + try { + this.animes = []; + this.$ = await redditocall('anime_index'); + return await this.parseRandom(); + } catch (err) { + console.log(err); + } + } + + async year(date: string) { + let animes: any = []; + + this.$ = await redditocall(date); + this.$('h3').each((index: number, element: cheerio.Element) => { + let parsed = this.parseAnime(this.$(element)); + parsed.year = date; + animes.push(parsed); + }); + return animes; + } + + parseRandom() { + return new Promise(async resolve => { + let data = this.$('p a'); + const origin: any = '1'; + let randomize = Math.round( + Math.random() * (data.length - 1 - origin) + parseInt(origin), + ); + + this.$ = await redditocall( + this.$('p a') + [randomize].attribs.href.split('/r/AnimeThemes/wiki/')[1] + .split('#wiki')[0], + ); + + let rand = Math.round(Math.random() * this.$('h3').length - 1); + let parsed = this.parseAnime(this.$('h3')[rand]); + resolve(parsed); + }); + } + + /* -ParseYears + Get the data from the year + get the name and the id to do the respective + scrapping. + */ + parseYears() { + return new Promise(async resolve => { + let years: any[] = []; + + this.$('h3 a').each((index: number, element: cheerio.Element) => { + years.push({ + id: this.$(element).attr('href').split('/')[4], + name: this.$(element).text(), + }); + }); + + resolve(years); + }); + } + + parseArtists() { + return new Promise(async resolve => { + let promises = []; + let data = this.$('p a').filter((x: any) => x > 0); + + for (let i = 0; i < data.length; i++) { + promises.push({ + id: data[i].children[0].parent.attribs.href.split('/')[5], + name: data[i].children[0].data, + }); + + if (i === data.length - 1) { + resolve(promises); + } + } + }); + } + + parseArtist() { + return new Promise(async resolve => { + let promises = []; + let data = this.$('h3'); + + for (let i = 0; i < data.length; i++) { + let parsed = await this.parseAnime(data[i]); + promises.push(parsed); + + if (i === data.length - 1) { + resolve(promises); + } + } + }); + } + + /* - ParseSerie + Parse the HTML from the redditocall + and search for the h3 tag to be the + same of the title and resolve a object. + */ + parseSerie(title: string) { + return new Promise(async resolve => { + let data = this.$('p a'); + + for (let i = 0; i < data.length; i++) { + let serieElement = data[i].children[0].data; + + if (serieElement.split(' (')[0] === title) { + let year = this.$('p a') + [i].attribs.href.split('/r/AnimeThemes/wiki/')[1] + .split('#wiki')[0]; + this.$ = await redditocall( + this.$('p a') + [i].attribs.href.split('/r/AnimeThemes/wiki/')[1] + .split('#wiki')[0], + ); + + for (let i = 0; i < this.$('h3').length; i++) { + if (this.$('h3')[i].children[0].children[0].data === title) { + let parsed = this.parseAnime(this.$('h3')[i]); + parsed.year = year; + resolve(parsed); + } + } + } + } + }); + } + + parseLinks() { + return new Promise(async resolve => { + let years = this.$('h3 a'); + + for (let i = 0; i < years.length; i++) { + let yearElement = years[i]; + + await this.year(this.$(yearElement).attr('href').split('/')[4]).then( + async animes => { + this.animes = this.animes.concat(animes); + if (i === years.length - 1) { + resolve(this.animes); + } + }, + ); + } + }); + } + + /* - ParseAnime + Parse the h3 tag and get the table + for the next function to parse the table + and get the information about the ending and + openings. + */ + parseAnime(element: cheerio.Element) { + let el = this.$(element).find('a'); + let title = this.$(el).text(); + let mal_id = this.$(el).attr('href').split('/')[4]; + let next = this.$(element).next(); + + let theme: any = { + id: mal_id, + title, + }; + + if (this.$(next).prop('tagName') === 'TABLE') { + theme.themes = this.parseTable(this.$(next)); + } else if (this.$(next).prop('tagName') === 'P') { + theme.themes = this.parseTable(this.$(next).next()); + } + + return theme; + } + + /* - ParseTable + Parse the table tag from the HTML + and returns a object with all the + information. + */ + parseTable(element: cheerio.Element): any { + if (this.$(element).prop('tagName') !== 'TABLE') { + return this.parseTable(this.$(element).next()); + } + + let themes: any = []; + + this.$(element) + .find('tbody') + .find('tr') + .each((index: number, element: cheerio.Element) => { + let name = replaceAll( + this.$(element).find('td').eq(0).text(), + '"', + '"', + ); + let link = this.$(element).find('td').eq(1).find('a').attr('href'); + let linkDesc = this.$(element).find('td').eq(1).find('a').text(); + let episodes = + this.$(element).find('td').eq(2).text().length > 0 + ? this.$(element).find('td').eq(2).text() + : ''; + let notes = + this.$(element).find('td').eq(3).text().length > 0 + ? this.$(element).find('td').eq(3).text() + : ''; + + themes.push({ + name, + link, + desc: linkDesc, + type: name.startsWith('OP') + ? 'Opening' + : name.startsWith('ED') + ? 'Ending' + : 'OP/ED', + episodes, + notes, + }); + }); + + return themes; + } +} + +async function redditocall(href: string) { + const resp = await requestGot(urls.REDDIT_ANIMETHEMES + href + '.json', { + parse: true, + scrapy: false, + }); + + return cheerio.load(getHTML(resp.data.content_html)); +} + +function getHTML(str: string) { + let html = replaceAll(str, '<', '<'); + html = replaceAll(html, '>', '>'); + return html; +} + +function replaceAll(str: string, find: string, replace: string) { + return str.replace(new RegExp(find, 'g'), replace); +} diff --git a/src/utils/util.ts b/src/utils/util.ts index d6b105e..6ecd1f6 100644 --- a/src/utils/util.ts +++ b/src/utils/util.ts @@ -409,3 +409,49 @@ async function desuServerUrl(url: string) { return result; } + +export const structureThemes = async (body: any, indv: boolean) => { + let themes: any[] = []; + + if (indv === true) { + return { + title: body.title, + year: body.year, + themes: await getThemesData(body.themes), + }; + } else { + for (let i = 0; i <= body.length - 1; i++) { + themes.push({ + title: body[i].title, + year: body[i].year, + themes: await getThemesData(body[i].themes), + }); + } + + return themes; + } +}; + +function getThemesData(themes: any[]): any { + let items: any[] = []; + + for (let i = 0; i <= themes.length - 1; i++) { + items.push({ + title: themes[i].name.split('"')[1] || 'Remasterización', + type: themes[i].type, + episodes: themes[i].episodes !== '' ? themes[i].episodes : null, + notes: themes[i].notes !== '' ? themes[i].notes : null, + video: themes[i].link, + }); + } + + return items.filter(x => x.title !== 'Remasterización'); +} + +export function getThemes(themes: any[]) { + return themes.map((item: any) => ({ + name: item.themeName, + type: item.themeType, + video: item.mirror.mirrorURL, + })); +} diff --git a/yarn.lock b/yarn.lock index bfec2d7..e3f33ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -174,6 +174,13 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c" integrity sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA== +"@types/redis@^2.8.28": + version "2.8.28" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.28.tgz#5862b2b64aa7f7cbc76dafd7e6f06992b52c55e3" + integrity sha512-8l2gr2OQ969ypa7hFOeKqtFoY70XkHxISV0pAwmQ2nm6CSPb1brmTmqJCGGrekCo+pAZyWlNXr+Kvo6L/1wijA== + dependencies: + "@types/node" "*" + "@types/responselike@*", "@types/responselike@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29" @@ -2317,6 +2324,33 @@ redent@^1.0.0: indent-string "^2.1.0" strip-indent "^1.0.1" +redis-commands@^1.5.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" + integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== + +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60= + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ= + dependencies: + redis-errors "^1.0.0" + +redis@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a" + integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ== + dependencies: + denque "^1.4.1" + redis-commands "^1.5.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + regexp-clone@1.0.0, regexp-clone@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/regexp-clone/-/regexp-clone-1.0.0.tgz#222db967623277056260b992626354a04ce9bf63"