Store nodeinfo per federated instances (#5578)

* Store nodeinfo per federated instances

* Update fetch-nodeinfo.ts

* Update fetch-nodeinfo.ts

* update

migration/1572760203493-nodeinfo.ts

+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class nodeinfo1572760203493 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "system"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "softwareName" character varying(64) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "softwareVersion" character varying(64) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "openRegistrations" boolean DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "name" character varying(256) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "description" character varying(4096) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerName" character varying(128) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerEmail" character varying(256) DEFAULT null`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "infoUpdatedAt" TIMESTAMP WITH TIME ZONE`, undefined);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "infoUpdatedAt"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerEmail"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerName"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "description"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "name"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "openRegistrations"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareVersion"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareName"`, undefined);
+        await queryRunner.query(`ALTER TABLE "instance" ADD "system" character varying(64)`, undefined);
+    }
+
+}

src/misc/app-lock.ts

@@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
 export function getApLock(uri: string, timeout = 30 * 1000) {
 	return lock(`ap-object:${uri}`, timeout);
 }
+
+export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
+	return lock(`nodeinfo:${host}`, timeout);
+}

src/models/entities/instance.ts

@@ -25,15 +25,6 @@ export class Instance {
 	})
 	public host: string;
 
-	/**
-	 * インスタンスのシステム (MastodonとかMisskeyとかPleromaとか)
-	 */
-	@Column('varchar', {
-		length: 64, nullable: true,
-		comment: 'The system of the Instance.'
-	})
-	public system: string | null;
-
 	/**
 	 * インスタンスのユーザー数
 	 */
@@ -129,4 +120,45 @@ export class Instance {
 		default: false
 	})
 	public isMarkedAsClosed: boolean;
+
+	@Column('varchar', {
+		length: 64, nullable: true, default: null,
+		comment: 'The software of the Instance.'
+	})
+	public softwareName: string | null;
+
+	@Column('varchar', {
+		length: 64, nullable: true, default: null,
+	})
+	public softwareVersion: string | null;
+
+	@Column('boolean', {
+		nullable: true, default: null,
+	})
+	public openRegistrations: boolean | null;
+
+	@Column('varchar', {
+		length: 256, nullable: true, default: null,
+	})
+	public name: string | null;
+
+	@Column('varchar', {
+		length: 4096, nullable: true, default: null,
+	})
+	public description: string | null;
+
+	@Column('varchar', {
+		length: 128, nullable: true, default: null,
+	})
+	public maintainerName: string | null;
+
+	@Column('varchar', {
+		length: 256, nullable: true, default: null,
+	})
+	public maintainerEmail: string | null;
+
+	@Column('timestamp with time zone', {
+		nullable: true,
+	})
+	public infoUpdatedAt: Date | null;
 }

src/queue/processors/deliver.ts

@@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
 import Logger from '../../services/logger';
 import { Instances } from '../../models';
 import { instanceChart } from '../../services/chart';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
 
 const logger = new Logger('deliver');
 
@@ -28,6 +29,8 @@ export default async (job: Bull.Job) => {
 				isNotResponding: false
 			});
 
+			fetchNodeinfo(i);
+
 			instanceChart.requestSent(i.host, true);
 		});
 

src/queue/processors/inbox.ts

@@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta';
 import { toPuny } from '../../misc/convert-host';
 import { validActor } from '../../remote/activitypub/type';
 import { ensure } from '../../prelude/ensure';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
 
 const logger = new Logger('inbox');
 
@@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise<void> => {
 			isNotResponding: false
 		});
 
+		fetchNodeinfo(i);
+
 		instanceChart.requestReceived(i.host);
 	});
 

src/remote/activitypub/models/person.ts

@@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type';
 import { getConnection } from 'typeorm';
 import { ensure } from '../../../prelude/ensure';
 import { toArray } from '../../../prelude/array';
+import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
 
 const logger = apLogger;
 
@@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
 	registerOrFetchInstanceDoc(host).then(i => {
 		Instances.increment({ id: i.id }, 'usersCount', 1);
 		instanceChart.newUser(i.host);
+		fetchNodeinfo(i);
 	});
 
 	usersChart.update(user!, true);

src/services/fetch-nodeinfo.ts

+import * as request from 'request-promise-native';
+import { Instance } from '../models/entities/instance';
+import { Instances } from '../models';
+import config from '../config';
+import { getNodeinfoLock } from '../misc/app-lock';
+import Logger from '../services/logger';
+
+export const logger = new Logger('nodeinfo', 'cyan');
+
+export async function fetchNodeinfo(instance: Instance) {
+	const unlock = await getNodeinfoLock(instance.host);
+
+	const _instance = await Instances.findOne({ host: instance.host });
+	const now = Date.now();
+	if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
+		unlock();
+		return;
+	}
+
+	logger.info(`Fetching nodeinfo of ${instance.host} ...`);
+
+	try {
+		const wellknown = await request({
+			url: 'https://' + instance.host + '/.well-known/nodeinfo',
+			proxy: config.proxy,
+			timeout: 1000 * 10,
+			forever: true,
+			headers: {
+				'User-Agent': config.userAgent,
+				Accept: 'application/json, */*'
+			},
+			json: true
+		}).catch(e => {
+			if (e.statusCode === 404) {
+				throw 'No nodeinfo provided';
+			} else {
+				throw e.statusCode || e.message;
+			}
+		});
+
+		if (wellknown.links == null || !Array.isArray(wellknown.links)) {
+			throw 'No wellknown links';
+		}
+
+		const links = wellknown.links as any[];
+
+		const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
+		const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
+		const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
+		const link = lnik2_1 || lnik2_0 || lnik1_0;
+
+		if (link == null) {
+			throw 'No nodeinfo link provided';
+		}
+
+		const info = await request({
+			url: link.href,
+			proxy: config.proxy,
+			timeout: 1000 * 10,
+			forever: true,
+			headers: {
+				'User-Agent': config.userAgent,
+				Accept: 'application/json, */*'
+			},
+			json: true
+		}).catch(e => {
+			throw e.statusCode || e.message;
+		});
+
+		await Instances.update(instance.id, {
+			infoUpdatedAt: new Date(),
+			softwareName: info.software.name.toLowerCase(),
+			softwareVersion: info.software.version,
+			openRegistrations: info.openRegistrations,
+			name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
+			description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
+			maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
+			maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
+		});
+
+		logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
+	} catch (e) {
+		logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
+
+		await Instances.update(instance.id, {
+			infoUpdatedAt: new Date(),
+		});
+	} finally {
+		unlock();
+	}
+}

src/services/register-or-fetch-instance-doc.ts

@@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
 			host,
 			caughtAt: new Date(),
 			lastCommunicatedAt: new Date(),
-			system: null // TODO
 		});
 
 		federationChart.update(true);