mirror of
https://github.com/MALSync/MAL-Sync-Backup.git
synced 2023-09-22 11:52:49 +02:00
Initial commit
This commit is contained in:
lolamtisch
commit
258c2e8ff1
6 changed files with 417 additions and 0 deletions
37
.github/workflows/backup.yml
vendored
Normal file
37
.github/workflows/backup.yml
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0'
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
name: Backup
|
||||
jobs:
|
||||
backup:
|
||||
name: Backup
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@master
|
||||
with:
|
||||
ref: master
|
||||
- uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: '14.x'
|
||||
- name: Build
|
||||
run: npm install
|
||||
- name: Backup
|
||||
run: npm run backup
|
||||
env:
|
||||
DOMAIN: ${{ secrets.DOMAIN }}
|
||||
ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
|
||||
- name: Commit files
|
||||
run: |
|
||||
git add data/*
|
||||
git add README.md
|
||||
git config --local user.email "action@github.com"
|
||||
git config --local user.name "GitHub Action"
|
||||
git commit -m "weekly backup" -a
|
||||
- name: Push changes
|
||||
uses: ad-m/github-push-action@master
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
branch: 'master'
|
||||
104
.gitignore
vendored
Normal file
104
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# TypeScript v1 declaration files
|
||||
typings/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Microbundle cache
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variables file
|
||||
.env
|
||||
.env.test
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and *not* Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
113
README.md
Normal file
113
README.md
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
# MAL-Sync Backup
|
||||
The main purppose of this repository is have a backup of the <a href="https://github.com/MALSync/MALSync">MAL-Sync</a> MAL to Streaming page mapping database. But using the data for other uses is allowed.
|
||||
The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated through all the users of MAL-Sync it should correct itself over time.
|
||||
|
||||
## Stats
|
||||
|
||||
<!--statstable-->
|
||||
| Page | Total | MalID | noMalID | AniID | noAniID |
|
||||
| --------- | ----- | ----- | ------- | ----- | ------- |
|
||||
| 9anime | 12558 | 12384 | 174 | 12041 | 517 |
|
||||
| Gogoanime | 7477 | 7369 | 108 | 7313 | 164 |
|
||||
| Mangadex | 45631 | 25354 | 20277 | 26655 | 18976 |
|
||||
| MangaNato | 4839 | 2678 | 2161 | 2928 | 1911 |
|
||||
| Twistmoe | 1992 | 1971 | 21 | 1966 | 26 |
|
||||
| animepahe | 3701 | 3699 | 2 | 3623 | 78 |
|
||||
| MangaFox | 6256 | 3891 | 2365 | 3861 | 2395 |
|
||||
| MangaSee | 5403 | 4565 | 838 | 4626 | 777 |
|
||||
<!--/statstable-->
|
||||
|
||||
## Structure
|
||||
|
||||
An _index.json can be found in every folder containing an array of all ids
|
||||
|
||||
### MAL -> Streaming Page Structure:
|
||||
`data/myanimelist/(anime|manga)/[id].json`
|
||||
|
||||
anime/19815
|
||||
```json
|
||||
{
|
||||
"altTitle": [
|
||||
"No Game, No Life",
|
||||
"NGNL",
|
||||
"ノーゲーム・ノーライフ"
|
||||
],
|
||||
"id": 19815,
|
||||
"type": "anime",
|
||||
"title": "No Game No Life",
|
||||
"url": "https://myanimelist.net/anime/19815/No_Game_No_Life",
|
||||
"image": "https://cdn.myanimelist.net/images/anime/5/65187.jpg",
|
||||
"category": "TV",
|
||||
"hentai": false,
|
||||
"createdAt": "2020-10-12T12:36:13.580Z",
|
||||
"updatedAt": "2020-10-15T11:36:06.203Z",
|
||||
"Pages": {
|
||||
"Aniwatch": {
|
||||
"350": {
|
||||
"...": "..."
|
||||
}
|
||||
},
|
||||
"9anime": {
|
||||
"4qkm": {
|
||||
"...": "..."
|
||||
},
|
||||
"y2p0": {
|
||||
"...": "..."
|
||||
}
|
||||
},
|
||||
"Gogoanime": {
|
||||
"no-game-no-life": {
|
||||
"...": "..."
|
||||
},
|
||||
"no-game-no-life-dub": {
|
||||
"...": "..."
|
||||
}
|
||||
},
|
||||
"Twistmoe": {
|
||||
"no-game-no-life": {
|
||||
"...": "..."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
### Streaming Page -> MAL Structure:
|
||||
`data/pages/[streaming page key]/[id].json`
|
||||
|
||||
9anime/214
|
||||
```json
|
||||
{
|
||||
"identifier": "214",
|
||||
"malUrl": "https://myanimelist.net/anime/9617/K-On_Movie",
|
||||
"type": "anime",
|
||||
"page": "9anime",
|
||||
"title": "K-On! Movie",
|
||||
"url": "...",
|
||||
"image": "....",
|
||||
"hentai": false,
|
||||
"sticky": false,
|
||||
"active": true,
|
||||
"actor": null,
|
||||
"malId": 9617,
|
||||
"createdAt": "...",
|
||||
"updatedAt": "...",
|
||||
"Mal": {
|
||||
"altTitle": [],
|
||||
"id": 9617,
|
||||
"type": "anime",
|
||||
"title": "K-On! Movie",
|
||||
"url": "...",
|
||||
"image": "...",
|
||||
"category": "-",
|
||||
"hentai": false,
|
||||
"createdAt": "...",
|
||||
"updatedAt": "..."
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
How to find the IDs can be checked in <a href="https://github.com/lolamtisch/MALSync/tree/master/src/pages">here</a>.
|
||||
`[PageKey]/main.ts -> (overview|sync):getIdentifier(url)`
|
||||
83
index.js
Normal file
83
index.js
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
const fetch = require('node-fetch');
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path');
|
||||
const table = require('markdown-table');
|
||||
|
||||
async function start() {
|
||||
fs.removeSync('data');
|
||||
await backup('/page/export', 'pages');
|
||||
await backup('/mal/export', 'myanimelist');
|
||||
await backup('/ani/export', 'anilist');
|
||||
await updateReadme();
|
||||
}
|
||||
|
||||
|
||||
async function backup(url, name, type = 'anime', page = 0, ids = {}) {
|
||||
await getData(url + '/' + type + '/' + page).then(async (json) => {
|
||||
const promises = json.data.map(data => {
|
||||
if(!ids[data.page ?? type]) {
|
||||
ids[data.page ?? type] = [];
|
||||
}
|
||||
ids[data.page ?? type].push(data.identifier ?? data.id);
|
||||
return fs.outputFile(`data/${name}/${data.page ?? type}/${data.identifier ?? data.id}.json`, JSON.stringify(data, null, 2))
|
||||
})
|
||||
await Promise.all(promises);
|
||||
|
||||
if(json.next) {
|
||||
return await backup(url, name, type, page + 1, ids);
|
||||
} else {
|
||||
for(item in ids) {
|
||||
await fs.outputFile(`data/${name}/${item}/_index.json`, JSON.stringify(ids[item].sort(), null, 2))
|
||||
}
|
||||
if(type === 'anime') {
|
||||
return await backup(url, name, 'manga')
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}).catch((e) => {throw 'backup error'});
|
||||
}
|
||||
|
||||
async function getData(url) {
|
||||
return fetch(process.env.DOMAIN + url, {
|
||||
headers: {
|
||||
'x-access-token': process.env.ACCESS_TOKEN
|
||||
}
|
||||
}).then(res => {
|
||||
if(res.status !== 200) {
|
||||
throw "response status not 200";
|
||||
}
|
||||
return res.json()
|
||||
})
|
||||
}
|
||||
|
||||
async function updateReadme() {
|
||||
getData('/stats/db').then(json => {
|
||||
let stats = [['Page', 'Total', 'MalID', 'noMalID', 'AniID', 'noAniID']];
|
||||
|
||||
for (const pageName in json.pages) {
|
||||
const page = json.pages[pageName];
|
||||
stats.push([pageName, page.total, page.mal, page.noMal, page.ani, page.noAni])
|
||||
}
|
||||
|
||||
const statstable = table(stats);
|
||||
const descFile = path.resolve('./README.md');
|
||||
|
||||
fs.readFile(descFile, 'utf8', function(err, data) {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
const result = data.replace(/<!--statstable-->((.|\n|\r)*)<!--\/statstable-->/g, `<!--statstable-->\n${statstable}\n<!--/statstable-->`);
|
||||
|
||||
fs.writeFile(descFile, result, 'utf8', function(err) {
|
||||
if (err) throw err;
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
process.on('unhandledRejection', err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
start();
|
||||
55
package-lock.json
generated
Normal file
55
package-lock.json
generated
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
{
|
||||
"name": "mal-sync-backup",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"fs-extra": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
|
||||
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.2.0",
|
||||
"jsonfile": "^6.0.1",
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.6",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
|
||||
"integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ=="
|
||||
},
|
||||
"jsonfile": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
|
||||
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.6",
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"markdown-table": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz",
|
||||
"integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==",
|
||||
"requires": {
|
||||
"repeat-string": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"repeat-string": {
|
||||
"version": "1.6.1",
|
||||
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
|
||||
"integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc="
|
||||
},
|
||||
"universalify": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
|
||||
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
|
||||
}
|
||||
}
|
||||
}
|
||||
25
package.json
Normal file
25
package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "mal-sync-backup",
|
||||
"version": "1.0.0",
|
||||
"description": "The main purppose of this repository is have a backup of the <a href=\"https://github.com/lolamtisch/MALSync\">MAL-Sync</a> MAL to Streaming page mapping database. But using the data for other uses is allowed. \r The data is updated once a week. Wrong/missing mappings are not seldom, specially mangas, but because they are generated throght all the users of MAL-Sync it should correct itself over time.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"backup": "node index.js",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/MALSync/MAL-Sync-Backup.git"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"bugs": {
|
||||
"url": "https://github.com/MALSync/MAL-Sync-Backup/issues"
|
||||
},
|
||||
"homepage": "https://github.com/MALSync/MAL-Sync-Backup#readme",
|
||||
"dependencies": {
|
||||
"fs-extra": "^10.0.0",
|
||||
"markdown-table": "^2.0.0",
|
||||
"node-fetch": "^2.6.1"
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue