mirror of
https://github.com/AdguardTeam/AdGuardHome.git
synced 2025-10-26 11:27:18 +00:00
Pull request 2508: rm-querylog-script
Squashed commit of the following:
commit 698fa2b7fd
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date: Wed Oct 22 11:30:07 2025 +0300
scripts: rm querylog
This commit is contained in:
Ainar Garipov
parent
9be4ca90ef
commit
36bc311141
@ -58,6 +58,10 @@ In this release, the schema version has changed from 30 to 31.
|
||||
|
||||
To roll back this change, set `schema_version` back to `30`.
|
||||
|
||||
### Removed
|
||||
|
||||
- The outdated querylog anonymization script.
|
||||
|
||||
[go-1.25.3]: https://groups.google.com/g/golang-announce/c/YEyj6FUNbik
|
||||
|
||||
<!--
|
||||
|
||||
@ -6,14 +6,6 @@
|
||||
|
||||
Run `make init` from the project root.
|
||||
|
||||
## `querylog/`: Query Log Helpers
|
||||
|
||||
### Usage
|
||||
|
||||
- `npm install`: install dependencies. Run this first.
|
||||
|
||||
- `npm run anonymize <source> <dst>`: read the query log from the `<source>` and write anonymized version to `<dst>`.
|
||||
|
||||
## `make/`: Makefile scripts
|
||||
|
||||
The release channels are: `development` (the default), `edge`, `beta`, and `release`. If verbosity levels aren’t documented here, there are only two: `0`, don’t print anything, and `1`, be verbose.
|
||||
|
||||
@ -1,118 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const readline = require('readline');
|
||||
const dnsPacket = require('dns-packet')
|
||||
|
||||
const processLineByLine = async (source, callback) => {
|
||||
const fileStream = fs.createReadStream(source);
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
|
||||
for await (const line of rl) {
|
||||
await callback(line);
|
||||
}
|
||||
}
|
||||
|
||||
const anonDomain = (domain) => {
|
||||
// Replace all question domain letters with a
|
||||
return domain.replace(/[a-z]/g, 'a');
|
||||
}
|
||||
|
||||
const anonIP = (ip) => {
|
||||
// Replace all numbers with '1'
|
||||
return ip.replace(/[0-9]/g, '1');
|
||||
}
|
||||
|
||||
const anonAnswer = (answer) => {
|
||||
const answerData = Buffer.from(answer, 'base64');
|
||||
const packet = dnsPacket.decode(answerData, 0);
|
||||
|
||||
packet.questions.forEach((q) => {
|
||||
q.name = anonDomain(q.name);
|
||||
});
|
||||
packet.answers.forEach((q) => {
|
||||
q.name = anonDomain(q.name);
|
||||
|
||||
if (q.type === 'A' || q.type === 'AAAA') {
|
||||
q.data = anonIP(q.data);
|
||||
} else if (typeof q.data === 'string') {
|
||||
q.data = anonDomain(q.data);
|
||||
}
|
||||
});
|
||||
|
||||
const anonData = dnsPacket.encode(packet);
|
||||
return anonData.toString('base64');
|
||||
}
|
||||
|
||||
const anonLine = (line) => {
|
||||
if (!line) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const logItem = JSON.parse(line);
|
||||
|
||||
// Replace all numbers with '1'
|
||||
logItem['IP'] = logItem['IP'].replace(/[0-9]/g, '1');
|
||||
// Replace all question domain letters with a
|
||||
logItem['QH'] = logItem['QH'].replace(/[a-z]/g, 'a');
|
||||
// Anonymize "Answer" and "OrigAnswer" fields
|
||||
if (logItem['Answer']) {
|
||||
logItem['Answer'] = anonAnswer(logItem['Answer']);
|
||||
}
|
||||
if (logItem['OrigAnswer']) {
|
||||
logItem['OrigAnswer'] = anonAnswer(logItem['OrigAnswer']);
|
||||
}
|
||||
|
||||
// If Result is set, anonymize the "Rule" field
|
||||
if (logItem['Result'] && logItem['Result']['Rule']) {
|
||||
logItem['Result']['Rule'] = anonDomain(logItem['Result']['Rule']);
|
||||
}
|
||||
|
||||
return JSON.stringify(logItem);
|
||||
} catch (ex) {
|
||||
console.error(`Failed to parse ${line}: ${ex} ${ex.stack}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const anon = async (source, dest) => {
|
||||
const out = fs.createWriteStream(dest, {
|
||||
flags: 'w',
|
||||
});
|
||||
|
||||
|
||||
await processLineByLine(source, async (line) => {
|
||||
const newLine = anonLine(line);
|
||||
if (!newLine) {
|
||||
return;
|
||||
}
|
||||
out.write(`${newLine}\n`);
|
||||
});
|
||||
}
|
||||
|
||||
const main = async () => {
|
||||
console.log('Start query log anonymization');
|
||||
|
||||
const source = process.argv[2];
|
||||
const dest = process.argv[3];
|
||||
|
||||
console.log(`Source: ${source}`);
|
||||
console.log(`Destination: ${dest}`);
|
||||
|
||||
if (!fs.existsSync(source)) {
|
||||
throw new Error(`${source} not found`);
|
||||
}
|
||||
|
||||
try {
|
||||
await anon(source, dest);
|
||||
} catch (ex) {
|
||||
console.error(ex);
|
||||
}
|
||||
|
||||
console.log('Finished query log anonymization')
|
||||
}
|
||||
|
||||
main();
|
||||
21
scripts/querylog/package-lock.json
generated
21
scripts/querylog/package-lock.json
generated
@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "querylog",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"dns-packet": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.2.1.tgz",
|
||||
"integrity": "sha512-JHj2yJeKOqlxzeuYpN1d56GfhzivAxavNwHj9co3qptECel27B1rLY5PifJAvubsInX5pGLDjAHuCfCUc2Zv/w==",
|
||||
"requires": {
|
||||
"ip": "^1.1.5"
|
||||
}
|
||||
},
|
||||
"ip": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
|
||||
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
{
|
||||
"name": "querylog",
|
||||
"version": "0.1.0",
|
||||
"scripts": {
|
||||
"anonymize": "node anonymize.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"dns-packet": "^5.2.1"
|
||||
}
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
{"IP":"192.168.0.0","T":"2020-08-31T16:43:37.724457416+03:00","QH":"mtalk.google.com","QT":"A","QC":"IN","CP":"","Answer":"rm+BgAABAAIAAAAABW10YWxrBmdvb2dsZQNjb20AAAEAAcAMAAUAAQAAnwUAEQxtb2JpbGUtZ3RhbGsBbMASwC4AAQABAAAAWQAEjvobvA==","Result":{},"Elapsed":48051030,"Upstream":"tls://unfiltered.adguard-dns.com:853"}
|
||||
{"IP":"127.0.0.1","T":"2020-09-09T13:56:35.532956+03:00","QH":"example.org","QT":"AAAA","QC":"IN","CP":"","Answer":"mrOBgAABAAEAAAAAB2V4YW1wbGUDb3JnAAAcAAHADAAcAAEAAKjAABAmBigAAiAAAQJIGJMlyBlG","Result":{},"Elapsed":132164793,"Upstream":"https://dns10.quad9.net:443/dns-query"}
|
||||
{"IP":"127.0.0.1","T":"2020-09-09T13:56:54.255453+03:00","QH":"ad.doubleclick.net","QT":"A","QC":"IN","CP":"","Answer":"wqmBgAABAAIAAAAAAmFkC2RvdWJsZWNsaWNrA25ldAAAAQABwAwABQABAACTawAJBGRhcnQBbMAPwDAAAQABAAAA5gAErNkQhg==","Result":{},"Elapsed":48131793,"Upstream":"https://dns10.quad9.net:443/dns-query"}
|
||||
{"IP":"127.0.0.1","T":"2020-09-09T13:57:07.495948+03:00","QH":"ad.doubleclick.net","QT":"A","QC":"IN","CP":"","Answer":"JP2BhQABAAAAAAAAAmFkC2RvdWJsZWNsaWNrA25ldAAAAQAB","Result":{"IsFiltered":true,"Reason":3,"Rule":"||ad.doubleclick.net^","FilterID":1},"Elapsed":369806}
|
||||
{"IP":"192.168.0.15","T":"2020-01-17T17:39:40.306375885+03:00","QH":"push.apple.com","QT":"TXT","QC":"IN","Answer":"8AWBgAABAAEAAAABBHB1c2gFYXBwbGUDY29tAAAQAAHADAAQAAEAABOsAAkIY291bnQ9NTAAACkFrAAAAAAAQAAMADwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","Result":{},"Elapsed":30271893,"Upstream":"https://cloudflare-dns.com:443/dns-query"}
|
||||
Loading…
Reference in New Issue
Block a user