diff --git a/package.json b/package.json
index 380c778a75fe925eea9650f519773d88497e8842..731565be8d9429e40fea0a16870ed3e6fdac5ad9 100644
--- a/package.json
+++ b/package.json
@@ -72,6 +72,7 @@
"@types/markdown-it": "0.0.9",
"@types/mocha": "7.0.2",
"@types/node": "13.11.0",
+ "@types/node-fetch": "2.5.5",
"@types/nodemailer": "6.4.0",
"@types/nprogress": "0.2.0",
"@types/oauth": "0.9.1",
@@ -84,8 +85,6 @@
"@types/ratelimiter": "2.1.28",
"@types/redis": "2.8.17",
"@types/rename": "1.0.1",
- "@types/request": "2.48.4",
- "@types/request-promise-native": "1.0.17",
"@types/request-stats": "3.0.0",
"@types/rimraf": "2.0.3",
"@types/seedrandom": "2.4.28",
@@ -102,7 +101,6 @@
"@types/websocket": "1.0.0",
"@types/ws": "7.2.3",
"@typescript-eslint/parser": "2.26.0",
- "agentkeepalive": "4.1.0",
"animejs": "3.1.0",
"apexcharts": "3.17.1",
"autobind-decorator": "2.4.0",
@@ -145,6 +143,7 @@
"gulp-typescript": "5.0.1",
"hard-source-webpack-plugin": "0.13.1",
"html-minifier": "4.0.0",
+ "http-proxy-agent": "4.0.1",
"http-signature": "1.3.4",
"https-proxy-agent": "5.0.0",
"insert-text-at-cursor": "0.3.0",
@@ -205,8 +204,6 @@
"redis-lock": "0.1.4",
"reflect-metadata": "0.1.13",
"rename": "1.0.4",
- "request": "2.88.2",
- "request-promise-native": "1.0.8",
"request-stats": "3.0.0",
"require-all": "3.0.0",
"rimraf": "3.0.2",
diff --git a/src/misc/donwload-url.ts b/src/misc/donwload-url.ts
index 939e6e980336697a5029e01d1b73e831ec476d4c..cd15bbd7311deaab6333255a7fc67ac048cb6463 100644
--- a/src/misc/donwload-url.ts
+++ b/src/misc/donwload-url.ts
@@ -1,5 +1,6 @@
import * as fs from 'fs';
-import * as request from 'request';
+import fetch from 'node-fetch';
+import { httpAgent, httpsAgent } from './fetch';
import config from '../config';
import * as chalk from 'chalk';
import Logger from '../services/logger';
@@ -7,53 +8,48 @@ import Logger from '../services/logger';
export async function downloadUrl(url: string, path: string) {
const logger = new Logger('download');
- await new Promise((res, rej) => {
- logger.info(`Downloading ${chalk.cyan(url)} ...`);
+ logger.info(`Downloading ${chalk.cyan(url)} ...`);
+
+ const response = await fetch(new URL(url).href, {
+ headers: {
+ 'User-Agent': config.userAgent
+ },
+ timeout: 10 * 1000,
+ agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+ }).then(response => {
+ if (!response.ok) {
+ logger.error(`Got ${response.status} (${url})`);
+ throw response.status;
+ } else {
+ return response;
+ }
+ });
+ await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
- writable.on('finish', () => {
- logger.succ(`Download finished: ${chalk.cyan(url)}`);
- res();
- });
-
- writable.on('error', error => {
- logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
+ response.body.on('error', (error: any) => {
+ logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
+ writable.close();
rej(error);
});
- const req = request({
- url: new URL(url).href, // https://github.com/syuilo/misskey/issues/2637
- proxy: config.proxy,
- timeout: 10 * 1000,
- forever: true,
- headers: {
- 'User-Agent': config.userAgent
- }
- });
-
- req.pipe(writable);
-
- req.on('response', response => {
- if (response.statusCode !== 200) {
- logger.error(`Got ${response.statusCode} (${url})`);
- writable.close();
- rej(response.statusCode);
- }
+ writable.on('finish', () => {
+ logger.succ(`Download finished: ${chalk.cyan(url)}`);
+ res();
});
- req.on('error', error => {
- logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
+ writable.on('error', error => {
+ logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
- writable.close();
rej(error);
});
- logger.succ(`Downloaded to: ${path}`);
+ response.body.pipe(writable);
});
}
diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts
new file mode 100644
index 0000000000000000000000000000000000000000..887aae1659637fa9421feb986e9cbcc2ce9a0d24
--- /dev/null
+++ b/src/misc/fetch.ts
@@ -0,0 +1,43 @@
+import * as http from 'http';
+import * as https from 'https';
+import * as cache from 'lookup-dns-cache';
+import fetch, { HeadersInit } from 'node-fetch';
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import config from '../config';
+
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
+ const res = await fetch(url, {
+ headers: Object.assign({
+ 'User-Agent': config.userAgent,
+ Accept: accept
+ }, headers || {}),
+ timeout,
+ agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
+ });
+
+ if (!res.ok) {
+ throw {
+ name: `StatusError`,
+ statusCode: res.status,
+ message: `${res.status} ${res.statusText}`,
+ };
+ }
+
+ return await res.json();
+}
+
+export const httpAgent = config.proxy
+ ? new HttpProxyAgent(config.proxy)
+ : new http.Agent({
+ keepAlive: true,
+ keepAliveMsecs: 30 * 1000,
+ });
+
+export const httpsAgent = config.proxy
+ ? new HttpsProxyAgent(config.proxy)
+ : new https.Agent({
+ keepAlive: true,
+ keepAliveMsecs: 30 * 1000,
+ lookup: cache.lookup,
+ });
diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts
index 0f87381a44a30465bf7b372d659d5a16f5e514c5..24540827b233a071a569a91cd12aad61b00cd4d7 100644
--- a/src/remote/activitypub/request.ts
+++ b/src/remote/activitypub/request.ts
@@ -1,19 +1,12 @@
import * as https from 'https';
import { sign } from 'http-signature';
import * as crypto from 'crypto';
-import * as cache from 'lookup-dns-cache';
import config from '../../config';
import { ILocalUser } from '../../models/entities/user';
import { UserKeypairs } from '../../models';
import { ensure } from '../../prelude/ensure';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-
-const agent = config.proxy
- ? new HttpsProxyAgent(config.proxy)
- : new https.Agent({
- lookup: cache.lookup,
- });
+import { httpsAgent } from '../../misc/fetch';
export default async (user: ILocalUser, url: string, object: any) => {
const timeout = 10 * 1000;
@@ -32,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => {
await new Promise((resolve, reject) => {
const req = https.request({
- agent,
+ agent: httpsAgent,
protocol,
hostname,
port,
diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts
index 8688b79a404e3ea20dbcd38838e7224afbab46d1..3847ea92ca97974b49410cf6bce1f703ce89a239 100644
--- a/src/remote/activitypub/resolver.ts
+++ b/src/remote/activitypub/resolver.ts
@@ -1,10 +1,8 @@
-import * as request from 'request-promise-native';
+import { getJson } from '../../misc/fetch';
import { IObject, isCollectionOrOrderedCollection, ICollection, IOrderedCollection } from './type';
-import config from '../../config';
export default class Resolver {
private history: Set<string>;
- private timeout = 10 * 1000;
constructor() {
this.history = new Set();
@@ -41,24 +39,7 @@ export default class Resolver {
this.history.add(value);
- const object = await request({
- url: value,
- proxy: config.proxy,
- timeout: this.timeout,
- forever: true,
- headers: {
- 'User-Agent': config.userAgent,
- Accept: 'application/activity+json, application/ld+json'
- },
- json: true
- }).catch(e => {
- const message = `${e.name}: ${e.message ? e.message.substr(0, 200) : undefined}, url=${value}`;
- throw {
- name: e.name,
- statusCode: e.statusCode,
- message,
- };
- });
+ const object = await getJson(value, 'application/activity+json, application/ld+json');
if (object == null || (
Array.isArray(object['@context']) ?
diff --git a/src/remote/webfinger.ts b/src/remote/webfinger.ts
index e19ef96a2fc30cf732d612cc830321ba5ee02124..04f978a35dd520a4b52c3dfcbbc77d0e86ba8c35 100644
--- a/src/remote/webfinger.ts
+++ b/src/remote/webfinger.ts
@@ -1,5 +1,4 @@
-import config from '../config';
-import * as request from 'request-promise-native';
+import { getJson } from '../misc/fetch';
import { query as urlQuery } from '../prelude/url';
type ILink = {
@@ -15,17 +14,7 @@ type IWebFinger = {
export default async function(query: string): Promise<IWebFinger> {
const url = genUrl(query);
- return await request({
- url,
- proxy: config.proxy,
- timeout: 10 * 1000,
- forever: true,
- headers: {
- 'User-Agent': config.userAgent,
- Accept: 'application/jrd+json, application/json'
- },
- json: true
- });
+ return await getJson(url, 'application/jrd+json, application/json');
}
function genUrl(query: string) {
diff --git a/src/server/api/service/discord.ts b/src/server/api/service/discord.ts
index c2bb02453bef2f4d9d5f7d418f75c54e07dd30db..a5ad18d99c16537d0dac959e8aaf15e4dbc8362e 100644
--- a/src/server/api/service/discord.ts
+++ b/src/server/api/service/discord.ts
@@ -1,6 +1,6 @@
import * as Koa from 'koa';
import * as Router from '@koa/router';
-import * as request from 'request';
+import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth';
import config from '../../../config';
import { publishMainStream } from '../../../services/stream';
@@ -174,20 +174,9 @@ router.get('/dc/cb', async ctx => {
}
}));
- const { id, username, discriminator } = await new Promise<any>((res, rej) =>
- request({
- url: 'https://discordapp.com/api/users/@me',
- headers: {
- 'Authorization': `Bearer ${accessToken}`,
- 'User-Agent': config.userAgent
- }
- }, (err, response, body) => {
- if (err) {
- rej(err);
- } else {
- res(JSON.parse(body));
- }
- }));
+ const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
+ 'Authorization': `Bearer ${accessToken}`,
+ });
if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session');
@@ -256,21 +245,9 @@ router.get('/dc/cb', async ctx => {
}
}));
- const { id, username, discriminator } = await new Promise<any>((res, rej) =>
- request({
- url: 'https://discordapp.com/api/users/@me',
- headers: {
- 'Authorization': `Bearer ${accessToken}`,
- 'User-Agent': config.userAgent
- }
- }, (err, response, body) => {
- if (err) {
- rej(err);
- } else {
- res(JSON.parse(body));
- }
- }));
-
+ const { id, username, discriminator } = await getJson('https://discordapp.com/api/users/@me', '*/*', 10 * 1000, {
+ 'Authorization': `Bearer ${accessToken}`,
+ });
if (!id || !username || !discriminator) {
ctx.throw(400, 'invalid session');
return;
diff --git a/src/server/api/service/github.ts b/src/server/api/service/github.ts
index e36c43ee38c5770718bbfacc15db78959b4c5821..663c3cc7545a5be048704a0707952b140803971c 100644
--- a/src/server/api/service/github.ts
+++ b/src/server/api/service/github.ts
@@ -1,6 +1,6 @@
import * as Koa from 'koa';
import * as Router from '@koa/router';
-import * as request from 'request';
+import { getJson } from '../../../misc/fetch';
import { OAuth2 } from 'oauth';
import config from '../../../config';
import { publishMainStream } from '../../../services/stream';
@@ -167,21 +167,9 @@ router.get('/gh/cb', async ctx => {
}
}));
- const { login, id } = await new Promise<any>((res, rej) =>
- request({
- url: 'https://api.github.com/user',
- headers: {
- 'Accept': 'application/vnd.github.v3+json',
- 'Authorization': `bearer ${accessToken}`,
- 'User-Agent': config.userAgent
- }
- }, (err, response, body) => {
- if (err)
- rej(err);
- else
- res(JSON.parse(body));
- }));
-
+ const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+ 'Authorization': `bearer ${accessToken}`
+ });
if (!login || !id) {
ctx.throw(400, 'invalid session');
return;
@@ -230,20 +218,9 @@ router.get('/gh/cb', async ctx => {
res({ accessToken });
}));
- const { login, id } = await new Promise<any>((res, rej) =>
- request({
- url: 'https://api.github.com/user',
- headers: {
- 'Accept': 'application/vnd.github.v3+json',
- 'Authorization': `bearer ${accessToken}`,
- 'User-Agent': config.userAgent
- }
- }, (err, response, body) => {
- if (err)
- rej(err);
- else
- res(JSON.parse(body));
- }));
+ const { login, id } = await getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+ 'Authorization': `bearer ${accessToken}`
+ });
if (!login || !id) {
ctx.throw(400, 'invalid session');
diff --git a/src/server/web/url-preview.ts b/src/server/web/url-preview.ts
index 2526ed0f83bc947d8bd693a023302081f83939ec..4dae6baafbe7e893ca133e7a64d05e699c6c519b 100644
--- a/src/server/web/url-preview.ts
+++ b/src/server/web/url-preview.ts
@@ -1,10 +1,10 @@
import * as Koa from 'koa';
-import * as request from 'request-promise-native';
import summaly from 'summaly';
import { fetchMeta } from '../../misc/fetch-meta';
import Logger from '../../services/logger';
import config from '../../config';
import { query } from '../../prelude/url';
+import { getJson } from '../../misc/fetch';
const logger = new Logger('url-preview');
@@ -16,15 +16,10 @@ module.exports = async (ctx: Koa.Context) => {
: `Getting preview of ${ctx.query.url}@${ctx.query.lang} ...`);
try {
- const summary = meta.summalyProxy ? await request.get({
- url: meta.summalyProxy,
- qs: {
- url: ctx.query.url,
- lang: ctx.query.lang || 'ja-JP'
- },
- forever: true,
- json: true
- }) : await summaly(ctx.query.url, {
+ const summary = meta.summalyProxy ? await getJson(`${meta.summalyProxy}?${query({
+ url: ctx.query.url,
+ lang: ctx.query.lang || 'ja-JP'
+ })}`) : await summaly(ctx.query.url, {
followRedirects: false,
lang: ctx.query.lang || 'ja-JP'
});
diff --git a/src/services/drive/s3.ts b/src/services/drive/s3.ts
index d136bb2694e758ace3c948855ce0c1beb20ecf09..2cbeef106de7e0513e6df7f415fc7090e4e97daf 100644
--- a/src/services/drive/s3.ts
+++ b/src/services/drive/s3.ts
@@ -1,32 +1,17 @@
import * as S3 from 'aws-sdk/clients/s3';
-import config from '../../config';
import { Meta } from '../../models/entities/meta';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import * as agentkeepalive from 'agentkeepalive';
-
-const httpsAgent = config.proxy
- ? new HttpsProxyAgent(config.proxy)
- : new agentkeepalive.HttpsAgent({
- freeSocketTimeout: 30 * 1000
- });
+import { httpsAgent, httpAgent } from '../../misc/fetch';
export function getS3(meta: Meta) {
- const conf = {
+ return new S3({
endpoint: meta.objectStorageEndpoint || undefined,
- accessKeyId: meta.objectStorageAccessKey,
- secretAccessKey: meta.objectStorageSecretKey,
+ accessKeyId: meta.objectStorageAccessKey!,
+ secretAccessKey: meta.objectStorageSecretKey!,
region: meta.objectStorageRegion || undefined,
sslEnabled: meta.objectStorageUseSSL,
s3ForcePathStyle: !!meta.objectStorageEndpoint,
httpOptions: {
+ agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent
}
- } as S3.ClientConfiguration;
-
- if (meta.objectStorageUseSSL) {
- conf.httpOptions!.agent = httpsAgent;
- }
-
- const s3 = new S3(conf);
-
- return s3;
+ });
}
diff --git a/src/services/fetch-nodeinfo.ts b/src/services/fetch-nodeinfo.ts
index e5d652a6b37ab9cf7239fd72211c42b6e448ce2e..0cf51e3377ff37557199200f48c95d1338946867 100644
--- a/src/services/fetch-nodeinfo.ts
+++ b/src/services/fetch-nodeinfo.ts
@@ -1,7 +1,6 @@
-import * as request from 'request-promise-native';
+import { getJson } from '../misc/fetch';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
-import config from '../config';
import { getNodeinfoLock } from '../misc/app-lock';
import Logger from '../services/logger';
@@ -20,23 +19,14 @@ export async function fetchNodeinfo(instance: Instance) {
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
- const wellknown = await request({
- url: 'https://' + instance.host + '/.well-known/nodeinfo',
- proxy: config.proxy,
- timeout: 1000 * 10,
- forever: true,
- headers: {
- 'User-Agent': config.userAgent,
- Accept: 'application/json, */*'
- },
- json: true
- }).catch(e => {
- if (e.statusCode === 404) {
- throw 'No nodeinfo provided';
- } else {
- throw e.statusCode || e.message;
- }
- });
+ const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
+ .catch(e => {
+ if (e.statusCode === 404) {
+ throw 'No nodeinfo provided';
+ } else {
+ throw e.statusCode || e.message;
+ }
+ });
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
@@ -53,19 +43,10 @@ export async function fetchNodeinfo(instance: Instance) {
throw 'No nodeinfo link provided';
}
- const info = await request({
- url: link.href,
- proxy: config.proxy,
- timeout: 1000 * 10,
- forever: true,
- headers: {
- 'User-Agent': config.userAgent,
- Accept: 'application/json, */*'
- },
- json: true
- }).catch(e => {
- throw e.statusCode || e.message;
- });
+ const info = await getJson(link.href)
+ .catch(e => {
+ throw e.statusCode || e.message;
+ });
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),
diff --git a/test/utils.ts b/test/utils.ts
index af190254ecd7b213967490f01dd591f9e40b3294..066bd33a56469389601eea0b7bd6002222d4b3ec 100644
--- a/test/utils.ts
+++ b/test/utils.ts
@@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as WebSocket from 'ws';
-const fetch = require('node-fetch');
-import * as req from 'request';
+import fetch from 'node-fetch';
+const FormData = require('form-data');
import * as childProcess from 'child_process';
export const async = (fn: Function) => (done: Function) => {
@@ -20,6 +20,9 @@ export const request = async (endpoint: string, params: any, me?: any): Promise<
try {
const res = await fetch('http://localhost:8080/api' + endpoint, {
method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ },
body: JSON.stringify(Object.assign(auth, params))
});
@@ -64,18 +67,23 @@ export const react = async (user: any, note: any, reaction: string): Promise<any
}, user);
};
-export const uploadFile = (user: any, path?: string): Promise<any> => new Promise((ok, rej) => {
- req.post({
- url: 'http://localhost:8080/api/drive/files/create',
- formData: {
- i: user.token,
- file: fs.createReadStream(path || __dirname + '/resources/Lenna.png')
- },
- json: true
- }, (err, httpResponse, body) => {
- ok(body);
- });
-});
+export const uploadFile = (user: any, path?: string): Promise<any> => {
+ const formData = new FormData();
+ formData.append('i', user.token);
+ formData.append('file', fs.createReadStream(path || __dirname + '/resources/Lenna.png'));
+
+ return fetch('http://localhost:8080/api/drive/files/create', {
+ method: 'post',
+ body: formData,
+ timeout: 30 * 1000,
+ }).then(res => {
+ if (!res.ok) {
+ throw `${res.status} ${res.statusText}`;
+ } else {
+ return res.json();
+ }
+ });
+};
export function connectStream(user: any, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
return new Promise((res, rej) => {