mirror of
https://github.com/MALSync/MAL-Sync-Backup.git
synced 2023-09-22 11:52:49 +02:00
Initial commit
This commit is contained in:
lolamtisch
commit
258c2e8ff1
6 changed files with 417 additions and 0 deletions
37
.github/workflows/backup.yml
vendored
Normal file
37
.github/workflows/backup.yml
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: '0 0 * * 0'
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
name: Backup
|
||||||
|
jobs:
|
||||||
|
backup:
|
||||||
|
name: Backup
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@master
|
||||||
|
with:
|
||||||
|
ref: master
|
||||||
|
- uses: actions/setup-node@v1
|
||||||
|
with:
|
||||||
|
node-version: '14.x'
|
||||||
|
- name: Build
|
||||||
|
run: npm install
|
||||||
|
- name: Backup
|
||||||
|
run: npm run backup
|
||||||
|
env:
|
||||||
|
DOMAIN: ${{ secrets.DOMAIN }}
|
||||||
|
ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
|
||||||
|
- name: Commit files
|
||||||
|
run: |
|
||||||
|
git add data/*
|
||||||
|
git add README.md
|
||||||
|
git config --local user.email "action@github.com"
|
||||||
|
git config --local user.name "GitHub Action"
|
||||||
|
git commit -m "weekly backup" -a
|
||||||
|
- name: Push changes
|
||||||
|
uses: ad-m/github-push-action@master
|
||||||
|
with:
|
||||||
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
branch: 'master'
|
||||||
104
.gitignore
vendored
Normal file
104
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
|
||||||
|
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||||
|
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||||
|
|
||||||
|
# Runtime data
|
||||||
|
pids
|
||||||
|
*.pid
|
||||||
|
*.seed
|
||||||
|
*.pid.lock
|
||||||
|
|
||||||
|
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||||
|
lib-cov
|
||||||
|
|
||||||
|
# Coverage directory used by tools like istanbul
|
||||||
|
coverage
|
||||||
|
*.lcov
|
||||||
|
|
||||||
|
# nyc test coverage
|
||||||
|
.nyc_output
|
||||||
|
|
||||||
|
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||||
|
.grunt
|
||||||
|
|
||||||
|
# Bower dependency directory (https://bower.io/)
|
||||||
|
bower_components
|
||||||
|
|
||||||
|
# node-waf configuration
|
||||||
|
.lock-wscript
|
||||||
|
|
||||||
|
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||||
|
build/Release
|
||||||
|
|
||||||
|
# Dependency directories
|
||||||
|
node_modules/
|
||||||
|
jspm_packages/
|
||||||
|
|
||||||
|
# TypeScript v1 declaration files
|
||||||
|
typings/
|
||||||
|
|
||||||
|
# TypeScript cache
|
||||||
|
*.tsbuildinfo
|
||||||
|
|
||||||
|
# Optional npm cache directory
|
||||||
|
.npm
|
||||||
|
|
||||||
|
# Optional eslint cache
|
||||||
|
.eslintcache
|
||||||
|
|
||||||
|
# Microbundle cache
|
||||||
|
.rpt2_cache/
|
||||||
|
.rts2_cache_cjs/
|
||||||
|
.rts2_cache_es/
|
||||||
|
.rts2_cache_umd/
|
||||||
|
|
||||||
|
# Optional REPL history
|
||||||
|
.node_repl_history
|
||||||
|
|
||||||
|
# Output of 'npm pack'
|
||||||
|
*.tgz
|
||||||
|
|
||||||
|
# Yarn Integrity file
|
||||||
|
.yarn-integrity
|
||||||
|
|
||||||
|
# dotenv environment variables file
|
||||||
|
.env
|
||||||
|
.env.test
|
||||||
|
|
||||||
|
# parcel-bundler cache (https://parceljs.org/)
|
||||||
|
.cache
|
||||||
|
|
||||||
|
# Next.js build output
|
||||||
|
.next
|
||||||
|
|
||||||
|
# Nuxt.js build / generate output
|
||||||
|
.nuxt
|
||||||
|
dist
|
||||||
|
|
||||||
|
# Gatsby files
|
||||||
|
.cache/
|
||||||
|
# Comment in the public line in if your project uses Gatsby and *not* Next.js
|
||||||
|
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||||
|
# public
|
||||||
|
|
||||||
|
# vuepress build output
|
||||||
|
.vuepress/dist
|
||||||
|
|
||||||
|
# Serverless directories
|
||||||
|
.serverless/
|
||||||
|
|
||||||
|
# FuseBox cache
|
||||||
|
.fusebox/
|
||||||
|
|
||||||
|
# DynamoDB Local files
|
||||||
|
.dynamodb/
|
||||||
|
|
||||||
|
# TernJS port file
|
||||||
|
.tern-port
|
||||||
113
README.md
Normal file
113
README.md
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
# MAL-Sync Backup
|
||||||
|
The main purppose of this repository is have a backup of the <a href="https://github.com/MALSync/MALSync">MAL-Sync</a> MAL to Streaming page mapping database. But using the data for other uses is allowed.
|
||||||
|
The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated through all the users of MAL-Sync it should correct itself over time.
|
||||||
|
|
||||||
|
## Stats
|
||||||
|
|
||||||
|
<!--statstable-->
|
||||||
|
| Page | Total | MalID | noMalID | AniID | noAniID |
|
||||||
|
| --------- | ----- | ----- | ------- | ----- | ------- |
|
||||||
|
| 9anime | 12558 | 12384 | 174 | 12041 | 517 |
|
||||||
|
| Gogoanime | 7477 | 7369 | 108 | 7313 | 164 |
|
||||||
|
| Mangadex | 45631 | 25354 | 20277 | 26655 | 18976 |
|
||||||
|
| MangaNato | 4839 | 2678 | 2161 | 2928 | 1911 |
|
||||||
|
| Twistmoe | 1992 | 1971 | 21 | 1966 | 26 |
|
||||||
|
| animepahe | 3701 | 3699 | 2 | 3623 | 78 |
|
||||||
|
| MangaFox | 6256 | 3891 | 2365 | 3861 | 2395 |
|
||||||
|
| MangaSee | 5403 | 4565 | 838 | 4626 | 777 |
|
||||||
|
<!--/statstable-->
|
||||||
|
|
||||||
|
## Structure
|
||||||
|
|
||||||
|
An _index.json can be found in every folder containing an array of all ids
|
||||||
|
|
||||||
|
### MAL -> Streaming Page Structure:
|
||||||
|
`data/myanimelist/(anime|manga)/[id].json`
|
||||||
|
|
||||||
|
anime/19815
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"altTitle": [
|
||||||
|
"No Game, No Life",
|
||||||
|
"NGNL",
|
||||||
|
"ノーゲーム・ノーライフ"
|
||||||
|
],
|
||||||
|
"id": 19815,
|
||||||
|
"type": "anime",
|
||||||
|
"title": "No Game No Life",
|
||||||
|
"url": "https://myanimelist.net/anime/19815/No_Game_No_Life",
|
||||||
|
"image": "https://cdn.myanimelist.net/images/anime/5/65187.jpg",
|
||||||
|
"category": "TV",
|
||||||
|
"hentai": false,
|
||||||
|
"createdAt": "2020-10-12T12:36:13.580Z",
|
||||||
|
"updatedAt": "2020-10-15T11:36:06.203Z",
|
||||||
|
"Pages": {
|
||||||
|
"Aniwatch": {
|
||||||
|
"350": {
|
||||||
|
"...": "..."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"9anime": {
|
||||||
|
"4qkm": {
|
||||||
|
"...": "..."
|
||||||
|
},
|
||||||
|
"y2p0": {
|
||||||
|
"...": "..."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Gogoanime": {
|
||||||
|
"no-game-no-life": {
|
||||||
|
"...": "..."
|
||||||
|
},
|
||||||
|
"no-game-no-life-dub": {
|
||||||
|
"...": "..."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Twistmoe": {
|
||||||
|
"no-game-no-life": {
|
||||||
|
"...": "..."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### Streaming Page -> MAL Structure:
|
||||||
|
`data/pages/[streaming page key]/[id].json`
|
||||||
|
|
||||||
|
9anime/214
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"identifier": "214",
|
||||||
|
"malUrl": "https://myanimelist.net/anime/9617/K-On_Movie",
|
||||||
|
"type": "anime",
|
||||||
|
"page": "9anime",
|
||||||
|
"title": "K-On! Movie",
|
||||||
|
"url": "...",
|
||||||
|
"image": "....",
|
||||||
|
"hentai": false,
|
||||||
|
"sticky": false,
|
||||||
|
"active": true,
|
||||||
|
"actor": null,
|
||||||
|
"malId": 9617,
|
||||||
|
"createdAt": "...",
|
||||||
|
"updatedAt": "...",
|
||||||
|
"Mal": {
|
||||||
|
"altTitle": [],
|
||||||
|
"id": 9617,
|
||||||
|
"type": "anime",
|
||||||
|
"title": "K-On! Movie",
|
||||||
|
"url": "...",
|
||||||
|
"image": "...",
|
||||||
|
"category": "-",
|
||||||
|
"hentai": false,
|
||||||
|
"createdAt": "...",
|
||||||
|
"updatedAt": "..."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
How to find the IDs can be checked in <a href="https://github.com/lolamtisch/MALSync/tree/master/src/pages">here</a>.
|
||||||
|
`[PageKey]/main.ts -> (overview|sync):getIdentifier(url)`
|
||||||
83
index.js
Normal file
83
index.js
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
const fetch = require('node-fetch');
|
||||||
|
const fs = require('fs-extra')
|
||||||
|
const path = require('path');
|
||||||
|
const table = require('markdown-table');
|
||||||
|
|
||||||
|
async function start() {
|
||||||
|
fs.removeSync('data');
|
||||||
|
await backup('/page/export', 'pages');
|
||||||
|
await backup('/mal/export', 'myanimelist');
|
||||||
|
await backup('/ani/export', 'anilist');
|
||||||
|
await updateReadme();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async function backup(url, name, type = 'anime', page = 0, ids = {}) {
|
||||||
|
await getData(url + '/' + type + '/' + page).then(async (json) => {
|
||||||
|
const promises = json.data.map(data => {
|
||||||
|
if(!ids[data.page ?? type]) {
|
||||||
|
ids[data.page ?? type] = [];
|
||||||
|
}
|
||||||
|
ids[data.page ?? type].push(data.identifier ?? data.id);
|
||||||
|
return fs.outputFile(`data/${name}/${data.page ?? type}/${data.identifier ?? data.id}.json`, JSON.stringify(data, null, 2))
|
||||||
|
})
|
||||||
|
await Promise.all(promises);
|
||||||
|
|
||||||
|
if(json.next) {
|
||||||
|
return await backup(url, name, type, page + 1, ids);
|
||||||
|
} else {
|
||||||
|
for(item in ids) {
|
||||||
|
await fs.outputFile(`data/${name}/${item}/_index.json`, JSON.stringify(ids[item].sort(), null, 2))
|
||||||
|
}
|
||||||
|
if(type === 'anime') {
|
||||||
|
return await backup(url, name, 'manga')
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}).catch((e) => {throw 'backup error'});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getData(url) {
|
||||||
|
return fetch(process.env.DOMAIN + url, {
|
||||||
|
headers: {
|
||||||
|
'x-access-token': process.env.ACCESS_TOKEN
|
||||||
|
}
|
||||||
|
}).then(res => {
|
||||||
|
if(res.status !== 200) {
|
||||||
|
throw "response status not 200";
|
||||||
|
}
|
||||||
|
return res.json()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updateReadme() {
|
||||||
|
getData('/stats/db').then(json => {
|
||||||
|
let stats = [['Page', 'Total', 'MalID', 'noMalID', 'AniID', 'noAniID']];
|
||||||
|
|
||||||
|
for (const pageName in json.pages) {
|
||||||
|
const page = json.pages[pageName];
|
||||||
|
stats.push([pageName, page.total, page.mal, page.noMal, page.ani, page.noAni])
|
||||||
|
}
|
||||||
|
|
||||||
|
const statstable = table(stats);
|
||||||
|
const descFile = path.resolve('./README.md');
|
||||||
|
|
||||||
|
fs.readFile(descFile, 'utf8', function(err, data) {
|
||||||
|
if (err) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const result = data.replace(/<!--statstable-->((.|\n|\r)*)<!--\/statstable-->/g, `<!--statstable-->\n${statstable}\n<!--/statstable-->`);
|
||||||
|
|
||||||
|
fs.writeFile(descFile, result, 'utf8', function(err) {
|
||||||
|
if (err) throw err;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('unhandledRejection', err => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
start();
|
||||||
55
package-lock.json
generated
Normal file
55
package-lock.json
generated
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
{
|
||||||
|
"name": "mal-sync-backup",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"lockfileVersion": 1,
|
||||||
|
"requires": true,
|
||||||
|
"dependencies": {
|
||||||
|
"fs-extra": {
|
||||||
|
"version": "10.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
|
||||||
|
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
|
||||||
|
"requires": {
|
||||||
|
"graceful-fs": "^4.2.0",
|
||||||
|
"jsonfile": "^6.0.1",
|
||||||
|
"universalify": "^2.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"graceful-fs": {
|
||||||
|
"version": "4.2.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
|
||||||
|
"integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ=="
|
||||||
|
},
|
||||||
|
"jsonfile": {
|
||||||
|
"version": "6.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
|
||||||
|
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
|
||||||
|
"requires": {
|
||||||
|
"graceful-fs": "^4.1.6",
|
||||||
|
"universalify": "^2.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"markdown-table": {
|
||||||
|
"version": "2.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz",
|
||||||
|
"integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==",
|
||||||
|
"requires": {
|
||||||
|
"repeat-string": "^1.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node-fetch": {
|
||||||
|
"version": "2.6.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||||
|
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||||
|
},
|
||||||
|
"repeat-string": {
|
||||||
|
"version": "1.6.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
|
||||||
|
"integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc="
|
||||||
|
},
|
||||||
|
"universalify": {
|
||||||
|
"version": "2.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
|
||||||
|
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
25
package.json
Normal file
25
package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
{
|
||||||
|
"name": "mal-sync-backup",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "The main purppose of this repository is have a backup of the <a href=\"https://github.com/lolamtisch/MALSync\">MAL-Sync</a> MAL to Streaming page mapping database. But using the data for other uses is allowed. \r The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated throght all the users of MAL-Sync it should correct itself over time.",
|
||||||
|
"main": "index.js",
|
||||||
|
"scripts": {
|
||||||
|
"backup": "node index.js",
|
||||||
|
"test": "echo \"Error: no test specified\" && exit 1"
|
||||||
|
},
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "git+https://github.com/MALSync/MAL-Sync-Backup.git"
|
||||||
|
},
|
||||||
|
"author": "",
|
||||||
|
"license": "ISC",
|
||||||
|
"bugs": {
|
||||||
|
"url": "https://github.com/MALSync/MAL-Sync-Backup/issues"
|
||||||
|
},
|
||||||
|
"homepage": "https://github.com/MALSync/MAL-Sync-Backup#readme",
|
||||||
|
"dependencies": {
|
||||||
|
"fs-extra": "^10.0.0",
|
||||||
|
"markdown-table": "^2.0.0",
|
||||||
|
"node-fetch": "^2.6.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue