From 258c2e8ff11348e252aa3f8eaa6baa0ec7eeb9e2 Mon Sep 17 00:00:00 2001 From: lolamtisch Date: Tue, 15 Jun 2021 09:52:18 +0200 Subject: [PATCH] Initial commit --- .github/workflows/backup.yml | 37 ++++++++++++ .gitignore | 104 ++++++++++++++++++++++++++++++++ README.md | 113 +++++++++++++++++++++++++++++++++++ index.js | 83 +++++++++++++++++++++++++ package-lock.json | 55 +++++++++++++++++ package.json | 25 ++++++++ 6 files changed, 417 insertions(+) create mode 100644 .github/workflows/backup.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 index.js create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml new file mode 100644 index 00000000000..65397203923 --- /dev/null +++ b/.github/workflows/backup.yml @@ -0,0 +1,37 @@ +on: + schedule: + - cron: '0 0 * * 0' + push: + branches: + - master +name: Backup +jobs: + backup: + name: Backup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + ref: master + - uses: actions/setup-node@v1 + with: + node-version: '14.x' + - name: Build + run: npm install + - name: Backup + run: npm run backup + env: + DOMAIN: ${{ secrets.DOMAIN }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + - name: Commit files + run: | + git add data/* + git add README.md + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git commit -m "weekly backup" -a + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: 'master' diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..67045665db2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and *not* Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port diff --git a/README.md b/README.md new file mode 100644 index 00000000000..73ed4ed8e67 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +# MAL-Sync Backup +The main purppose of this repository is have a backup of the MAL-Sync MAL to Streaming page mapping database. But using the data for other uses is allowed. +The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated through all the users of MAL-Sync it should correct itself over time. + +## Stats + + +| Page | Total | MalID | noMalID | AniID | noAniID | +| --------- | ----- | ----- | ------- | ----- | ------- | +| 9anime | 12558 | 12384 | 174 | 12041 | 517 | +| Gogoanime | 7477 | 7369 | 108 | 7313 | 164 | +| Mangadex | 45631 | 25354 | 20277 | 26655 | 18976 | +| MangaNato | 4839 | 2678 | 2161 | 2928 | 1911 | +| Twistmoe | 1992 | 1971 | 21 | 1966 | 26 | +| animepahe | 3701 | 3699 | 2 | 3623 | 78 | +| MangaFox | 6256 | 3891 | 2365 | 3861 | 2395 | +| MangaSee | 5403 | 4565 | 838 | 4626 | 777 | + + +## Structure + +An _index.json can be found in every folder containing an array of all ids + +### MAL -> Streaming Page Structure: +`data/myanimelist/(anime|manga)/[id].json` + +anime/19815 +```json +{ + "altTitle": [ + "No Game, No Life", + "NGNL", + "ノーゲーム・ノーライフ" + ], + "id": 19815, + "type": "anime", + "title": "No Game No Life", + "url": "https://myanimelist.net/anime/19815/No_Game_No_Life", + "image": "https://cdn.myanimelist.net/images/anime/5/65187.jpg", + "category": "TV", + "hentai": false, + "createdAt": "2020-10-12T12:36:13.580Z", + "updatedAt": "2020-10-15T11:36:06.203Z", + "Pages": { + "Aniwatch": { + "350": { + "...": "..." + } + }, + "9anime": { + "4qkm": { + "...": "..." + }, + "y2p0": { + "...": "..." + } + }, + "Gogoanime": { + "no-game-no-life": { + "...": "..." + }, + "no-game-no-life-dub": { + "...": "..." + } + }, + "Twistmoe": { + "no-game-no-life": { + "...": "..." + } + } + } +} + +``` + +### Streaming Page -> MAL Structure: +`data/pages/[streaming page key]/[id].json` + +9anime/214 +```json +{ + "identifier": "214", + "malUrl": "https://myanimelist.net/anime/9617/K-On_Movie", + "type": "anime", + "page": "9anime", + "title": "K-On! Movie", + "url": "...", + "image": "....", + "hentai": false, + "sticky": false, + "active": true, + "actor": null, + "malId": 9617, + "createdAt": "...", + "updatedAt": "...", + "Mal": { + "altTitle": [], + "id": 9617, + "type": "anime", + "title": "K-On! Movie", + "url": "...", + "image": "...", + "category": "-", + "hentai": false, + "createdAt": "...", + "updatedAt": "..." + } +} + +``` + +How to find the IDs can be checked in here. +`[PageKey]/main.ts -> (overview|sync):getIdentifier(url)` diff --git a/index.js b/index.js new file mode 100644 index 00000000000..9b4c83ef6dd --- /dev/null +++ b/index.js @@ -0,0 +1,83 @@ +const fetch = require('node-fetch'); +const fs = require('fs-extra') +const path = require('path'); +const table = require('markdown-table'); + +async function start() { + fs.removeSync('data'); + await backup('/page/export', 'pages'); + await backup('/mal/export', 'myanimelist'); + await backup('/ani/export', 'anilist'); + await updateReadme(); +} + + +async function backup(url, name, type = 'anime', page = 0, ids = {}) { + await getData(url + '/' + type + '/' + page).then(async (json) => { + const promises = json.data.map(data => { + if(!ids[data.page ?? type]) { + ids[data.page ?? type] = []; + } + ids[data.page ?? type].push(data.identifier ?? data.id); + return fs.outputFile(`data/${name}/${data.page ?? type}/${data.identifier ?? data.id}.json`, JSON.stringify(data, null, 2)) + }) + await Promise.all(promises); + + if(json.next) { + return await backup(url, name, type, page + 1, ids); + } else { + for(item in ids) { + await fs.outputFile(`data/${name}/${item}/_index.json`, JSON.stringify(ids[item].sort(), null, 2)) + } + if(type === 'anime') { + return await backup(url, name, 'manga') + } + return true; + } + }).catch((e) => {throw 'backup error'}); +} + +async function getData(url) { + return fetch(process.env.DOMAIN + url, { + headers: { + 'x-access-token': process.env.ACCESS_TOKEN + } + }).then(res => { + if(res.status !== 200) { + throw "response status not 200"; + } + return res.json() + }) +} + +async function updateReadme() { + getData('/stats/db').then(json => { + let stats = [['Page', 'Total', 'MalID', 'noMalID', 'AniID', 'noAniID']]; + + for (const pageName in json.pages) { + const page = json.pages[pageName]; + stats.push([pageName, page.total, page.mal, page.noMal, page.ani, page.noAni]) + } + + const statstable = table(stats); + const descFile = path.resolve('./README.md'); + + fs.readFile(descFile, 'utf8', function(err, data) { + if (err) { + throw err; + } + const result = data.replace(/((.|\n|\r)*)/g, `\n${statstable}\n`); + + fs.writeFile(descFile, result, 'utf8', function(err) { + if (err) throw err; + }); + }); + }); +} + +process.on('unhandledRejection', err => { + console.error(err); + process.exit(1); +}); + +start(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000..8dfe1c954ff --- /dev/null +++ b/package-lock.json @@ -0,0 +1,55 @@ +{ + "name": "mal-sync-backup", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "graceful-fs": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==" + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "markdown-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", + "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", + "requires": { + "repeat-string": "^1.0.0" + } + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000000..97ad8cea202 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "mal-sync-backup", + "version": "1.0.0", + "description": "The main purppose of this repository is have a backup of the MAL-Sync MAL to Streaming page mapping database. But using the data for other uses is allowed. \r The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated throght all the users of MAL-Sync it should correct itself over time.", + "main": "index.js", + "scripts": { + "backup": "node index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/MALSync/MAL-Sync-Backup.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/MALSync/MAL-Sync-Backup/issues" + }, + "homepage": "https://github.com/MALSync/MAL-Sync-Backup#readme", + "dependencies": { + "fs-extra": "^10.0.0", + "markdown-table": "^2.0.0", + "node-fetch": "^2.6.1" + } +}