Merge branch 'develop' of github.com:thedevs-network/kutt into develop

This commit is contained in:
poeti8 2019-11-19 20:37:59 +03:30
commit edd424535b
11 changed files with 262 additions and 73 deletions

View File

@ -17,7 +17,6 @@ NEO4J_DB_USERNAME=neo4j
NEO4J_DB_PASSWORD=BjEphmupAf1D5pDD
# Redis host and port
REDIS_DISABLED=false
REDIS_HOST="127.0.0.1"
REDIS_PORT=6379
REDIS_PASSWORD=
@ -33,7 +32,7 @@ NON_USER_COOLDOWN=0
DEFAULT_MAX_STATS_PER_LINK=5000
# Use HTTPS for links with custom domain
CUSTOM_DOMAIN_USE_HTTPS=false
CUSTOM_DOMAIN_USE_HTTPS=false
# A passphrase to encrypt JWT. Use a long and secure key.
JWT_SECRET=securekey
@ -74,4 +73,4 @@ MAIL_PASSWORD=
REPORT_MAIL=
# Support email to show on the app
CONTACT_EMAIL=
CONTACT_EMAIL=

166
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "kutt",
"version": "2.0.1",
"version": "2.1.5",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -1258,6 +1258,15 @@
"@types/node": "*"
}
},
"@types/bull": {
"version": "3.10.5",
"resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.10.5.tgz",
"integrity": "sha512-1/4/tNitRNBjXpQ9tJdWJxpbVft0fFUPc5d0UPB/1NJgxn8Wd9xjSkFkS+YxwUqW6+NfFzOFqqPMPL+yzLL6LQ==",
"dev": true,
"requires": {
"@types/ioredis": "*"
}
},
"@types/connect": {
"version": "3.4.32",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.32.tgz",
@ -1337,6 +1346,15 @@
"@types/express": "*"
}
},
"@types/ioredis": {
"version": "4.0.18",
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.0.18.tgz",
"integrity": "sha512-iDIRGPGP4LwoeiKNxQcI38ZA5T8SC+MbGCiiNFJ+LNy9tdegj6f9PAZ7se4tiWJhUHbf25kEJt7k3YfmYjWKZg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/json-schema": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.3.tgz",
@ -3690,6 +3708,35 @@
"resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
"integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug="
},
"bull": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.11.0.tgz",
"integrity": "sha512-QQOn63RkL6CfnmZcacPVg1EF42SwQcYxNSn9OGlM5S2JW+Gah/dwCcXxZQ3h2nYnhsNfBsherJ7EpLzIsi2kSQ==",
"requires": {
"cron-parser": "^2.13.0",
"debuglog": "^1.0.0",
"get-port": "^5.0.0",
"ioredis": "^4.14.1",
"lodash": "^4.17.15",
"p-timeout": "^3.1.0",
"promise.prototype.finally": "^3.1.1",
"semver": "^6.3.0",
"util.promisify": "^1.0.0",
"uuid": "^3.3.3"
},
"dependencies": {
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
},
"uuid": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz",
"integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ=="
}
}
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -3978,6 +4025,11 @@
}
}
},
"cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw=="
},
"collection-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@ -4299,6 +4351,15 @@
"sha.js": "^2.4.8"
}
},
"cron-parser": {
"version": "2.13.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.13.0.tgz",
"integrity": "sha512-UWeIpnRb0eyoWPVk+pD3TDpNx3KCFQeezO224oJIkktBrcW6RoAPOx5zIKprZGfk6vcYSmA8yQXItejSaDBhbQ==",
"requires": {
"is-nan": "^1.2.1",
"moment-timezone": "^0.5.25"
}
},
"cross-spawn": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
@ -4484,6 +4545,11 @@
"ms": "^2.1.1"
}
},
"debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI="
},
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
@ -4606,6 +4672,11 @@
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"denque": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@ -6470,6 +6541,14 @@
"integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=",
"dev": true
},
"get-port": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.0.0.tgz",
"integrity": "sha512-imzMU0FjsZqNa6BqOjbbW6w5BivHIuQKopjpPqcnx0AVHJQKCxK1O+Ab3OrVXhrekqfVMjwA9ZYu062R+KcIsQ==",
"requires": {
"type-fest": "^0.3.0"
}
},
"get-stdin": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
@ -7098,6 +7177,40 @@
"loose-envify": "^1.0.0"
}
},
"ioredis": {
"version": "4.14.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz",
"integrity": "sha512-94W+X//GHM+1GJvDk6JPc+8qlM7Dul+9K+lg3/aHixPN7ZGkW6qlvX0DG6At9hWtH2v3B32myfZqWoANUJYGJA==",
"requires": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.1.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"redis-commands": "1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.0.1"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
}
}
},
"ip-address": {
"version": "5.9.4",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-5.9.4.tgz",
@ -7281,6 +7394,14 @@
"is-path-inside": "^1.0.0"
}
},
"is-nan": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.2.1.tgz",
"integrity": "sha1-n69ltvttskt/XAYoR16nH5iEAeI=",
"requires": {
"define-properties": "^1.1.1"
}
},
"is-npm": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz",
@ -7757,6 +7878,16 @@
"resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
"integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168="
},
"lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
},
"lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8="
},
"lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
@ -8153,6 +8284,14 @@
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
},
"moment-timezone": {
"version": "0.5.27",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.27.tgz",
"integrity": "sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw==",
"requires": {
"moment": ">= 2.9.0"
}
},
"morgan": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/morgan/-/morgan-1.9.1.tgz",
@ -9530,6 +9669,16 @@
"resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
"integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM="
},
"promise.prototype.finally": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.1.tgz",
"integrity": "sha512-gnt8tThx0heJoI3Ms8a/JdkYBVhYP/wv+T7yQimR+kdOEJL21xTFbiJhMRqnSPcr54UVvMbsscDk2w+ivyaLPw==",
"requires": {
"define-properties": "^1.1.3",
"es-abstract": "^1.13.0",
"function-bind": "^1.1.1"
}
},
"prop-types": {
"version": "15.7.2",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
@ -10025,6 +10174,11 @@
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
"integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
@ -10852,6 +11006,11 @@
"resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.0.4.tgz",
"integrity": "sha512-to7oADIniaYwS3MhtCa/sQhrxidCCQiF/qp4/m5iN3ipf0Y7Xlri0f6eG29r08aL7JYl8n32AF3Q5GYBZ7K8vw=="
},
"standard-as-callback": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz",
"integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg=="
},
"static-extend": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
@ -11306,6 +11465,11 @@
"integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
"dev": true
},
"type-fest": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz",
"integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ=="
},
"type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",

View File

@ -1,6 +1,6 @@
{
"name": "kutt",
"version": "2.0.1",
"version": "2.1.5",
"description": "Modern URL shortener.",
"main": "./production-server/server.js",
"scripts": {
@ -34,6 +34,7 @@
"dependencies": {
"axios": "^0.19.0",
"bcryptjs": "^2.4.3",
"bull": "^3.11.0",
"cookie-parser": "^1.4.4",
"cors": "^2.8.5",
"date-fns": "^2.4.1",
@ -91,6 +92,7 @@
"@babel/register": "^7.0.0",
"@types/bcryptjs": "^2.4.2",
"@types/body-parser": "^1.17.0",
"@types/bull": "^3.10.5",
"@types/cookie-parser": "^1.4.1",
"@types/cors": "^2.8.5",
"@types/date-fns": "^2.6.0",

View File

@ -136,7 +136,6 @@ export const signup: Handler = async (req, res) => {
if (email.length > 255) {
return res.status(400).json({ error: "Maximum email length is 255." });
}
const user = await getUser(email);
if (user && user.verified) {

View File

@ -1,21 +1,17 @@
import bcrypt from "bcryptjs";
import dns from "dns";
import { Handler } from "express";
import geoip from "geoip-lite";
import isbot from "isbot";
import generate from "nanoid/generate";
import ua from "universal-analytics";
import URL from "url";
import urlRegex from "url-regex";
import useragent from "useragent";
import { promisify } from "util";
import { deleteDomain, getDomain, setDomain } from "../db/domain";
import { addIP } from "../db/ip";
import {
addLinkCount,
banLink,
createShortLink,
createVisit,
deleteLink,
findLink,
getLinks,
@ -24,12 +20,7 @@ import {
} from "../db/link";
import transporter from "../mail/mail";
import * as redis from "../redis";
import {
addProtocol,
generateShortLink,
getStatsCacheTime,
getStatsLimit
} from "../utils";
import { addProtocol, generateShortLink, getStatsCacheTime } from "../utils";
import {
checkBannedDomain,
checkBannedHost,
@ -38,6 +29,7 @@ import {
preservedUrls,
urlCountsCheck
} from "./validateBodyController";
import { visitQueue } from "../queues";
const dnsLookup = promisify(dns.lookup);
@ -119,29 +111,17 @@ export const shortener: Handler = async (req, res) => {
}
};
const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
const filterInBrowser = agent => item =>
agent.family.toLowerCase().includes(item.toLocaleLowerCase());
const filterInOs = agent => item =>
agent.os.family.toLowerCase().includes(item.toLocaleLowerCase());
export const goToLink: Handler = async (req, res, next) => {
const { host } = req.headers;
const reqestedId = req.params.id || req.body.id;
const address = reqestedId.replace("+", "");
const customDomain = host !== process.env.DEFAULT_DOMAIN && host;
// TODO: Extract parsing into their own function
const agent = useragent.parse(req.headers["user-agent"]);
const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
const [os = "Other"] = osList.filter(filterInOs(agent));
const referrer =
req.header("Referer") && URL.parse(req.header("Referer")).hostname;
const location = geoip.lookup(req.realIP);
const country = location && location.country;
const isBot = isbot(req.headers["user-agent"]);
const domain = await (customDomain && getDomain({ address: customDomain }));
let domain;
if (customDomain) {
domain = await getDomain({ address: customDomain });
}
const link = await findLink({ address, domain_id: domain && domain.id });
@ -176,29 +156,24 @@ export const goToLink: Handler = async (req, res, next) => {
return res.status(401).json({ error: "Password is not correct" });
}
if (link.user_id && !isBot) {
addLinkCount(link.id);
createVisit({
browser: browser.toLowerCase(),
country: country || "Unknown",
domain: customDomain,
id: link.id,
os: os.toLowerCase().replace(/\s/gi, ""),
referrer: referrer.replace(/\./gi, "[dot]") || "Direct",
limit: getStatsLimit()
visitQueue.add({
headers: req.headers,
realIP: req.realIP,
referrer: req.get("Referrer"),
link,
customDomain
});
}
return res.status(200).json({ target: link.target });
}
if (link.user_id && !isBot) {
addLinkCount(link.id);
createVisit({
browser: browser.toLowerCase(),
country: (country && country.toLocaleLowerCase()) || "unknown",
domain: customDomain,
id: link.id,
os: os.toLowerCase().replace(/\s/gi, ""),
referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "direct",
limit: getStatsLimit()
visitQueue.add({
headers: req.headers,
realIP: req.realIP,
referrer: req.get("Referrer"),
link,
customDomain
});
}

View File

@ -59,7 +59,6 @@ interface ICreateVisit {
country: string;
domain?: string;
id: number;
limit: number;
os: string;
referrer: string;
}
@ -399,7 +398,6 @@ export const getStats = async (link: Link, domain: Domain) => {
set(new Date(), { date: 1 }),
set(new Date(visit.created_at), { date: 1 })
);
console.log(diff);
const index = stats.allTime.views.length - diff - 1;
const view = stats.allTime.views[index];
stats.allTime.stats = {

1
server/queues/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./queues";

20
server/queues/queues.ts Normal file
View File

@ -0,0 +1,20 @@
import Queue from "bull";
import path from "path";
const redis = {
port: Number(process.env.REDIS_PORT) || 6379,
host: process.env.REDIS_HOST || "127.0.0.1",
...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD })
};
const removeJob = job => job.remove();
export const visitQueue = new Queue("visit", { redis });
visitQueue.clean(5000, "completed");
visitQueue.clean(5000, "failed");
visitQueue.process(4, path.resolve(__dirname, "visitQueue.js"));
visitQueue.on("completed", removeJob);
visitQueue.on("failed", removeJob);

View File

@ -0,0 +1,40 @@
import useragent from "useragent";
import geoip from "geoip-lite";
import URL from "url";
import { createVisit, addLinkCount } from "../db/link";
import { getStatsLimit } from "../utils";
const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
const filterInBrowser = agent => item =>
agent.family.toLowerCase().includes(item.toLocaleLowerCase());
const filterInOs = agent => item =>
agent.os.family.toLowerCase().includes(item.toLocaleLowerCase());
export default function({ data }) {
const tasks = [];
tasks.push(addLinkCount(data.link.id));
if (data.link.visit_count < getStatsLimit()) {
const agent = useragent.parse(data.headers["user-agent"]);
const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
const [os = "Other"] = osList.filter(filterInOs(agent));
const referrer = data.referrer && URL.parse(data.referrer).hostname;
const location = geoip.lookup(data.realIP);
const country = location && location.country;
tasks.push(
createVisit({
browser: browser.toLowerCase(),
country: country || "Unknown",
domain: data.customDomain,
id: data.link.id,
os: os.toLowerCase().replace(/\s/gi, ""),
referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "Direct"
})
);
}
return Promise.all(tasks);
}

View File

@ -1,31 +1,23 @@
import { promisify } from "util";
import redis from "redis";
const disabled = process.env.REDIS_DISABLED === "true";
const client = redis.createClient({
host: process.env.REDIS_HOST || "127.0.0.1",
port: Number(process.env.REDIS_PORT) || 6379,
...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD })
});
const client =
!disabled &&
redis.createClient({
host: process.env.REDIS_HOST || "127.0.0.1",
port: Number(process.env.REDIS_PORT) || 6379,
...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD })
});
const defaultResolver: () => Promise<null> = () => Promise.resolve(null);
export const get: (key: string) => Promise<any> = disabled
? defaultResolver
: promisify(client.get).bind(client);
export const get: (key: string) => Promise<any> = promisify(client.get).bind(
client
);
export const set: (
key: string,
value: string,
ex?: string,
exValue?: number
) => Promise<any> = disabled
? defaultResolver
: promisify(client.set).bind(client);
) => Promise<any> = promisify(client.set).bind(client);
export const del: (key: string) => Promise<any> = disabled
? defaultResolver
: promisify(client.del).bind(client);
export const del: (key: string) => Promise<any> = promisify(client.del).bind(
client
);

View File

@ -56,7 +56,6 @@ app.prepare().then(async () => {
server.use(express.static("static"));
server.use((error, req, res, next) => {
console.log({ error });
res
.status(500)
.json({ error: "Sorry an error ocurred. Please try again later." });