change KV emulation to leveldb, remove redis

This commit is contained in:
41666 2020-12-13 21:30:55 -05:00
parent 3bca07c7d7
commit 8fe3d1dcf2
7 changed files with 195 additions and 76 deletions

1
.gitignore vendored
View file

@ -5,3 +5,4 @@ storybook-static
.next
worker
wrangler.toml
.devdbs

View file

@ -8,8 +8,6 @@ services:
- '.:/src'
ports:
- 6609:6609
links:
- redis
working_dir: /src
command: yarn worker
@ -21,8 +19,3 @@ services:
- 6601:6601
working_dir: /src
command: yarn ui
redis:
image: redis:6-alpine
ports:
- 6379

View file

@ -21,4 +21,4 @@ FROM mhart/alpine-node:slim-14 AS output
COPY --from=builder /src .
ENV PORT=6601
ENTRYPOINT [ "/bin/sh", "-c", "/usr/bin/node node_modules/.bin/next start -p $PORT"]
ENTRYPOINT [ "/bin/sh", "-c", "/usr/bin/node node_modules/.bin/next start -p $PORT" ]

View file

@ -88,13 +88,13 @@
"jest-enzyme": "^7.1.2",
"jest-react-hooks-shallow": "^1.4.2",
"jest-styled-components": "^7.0.3",
"level": "^6.0.1",
"minimist": "^1.2.5",
"npm-run-all": "^4.1.5",
"prettier": "^2.2.1",
"prettier-plugin-organize-imports": "^1.1.1",
"prettier-plugin-pkg": "^0.8.0",
"prettier-plugin-sh": "^0.6.0",
"redis": "^3.0.2",
"stylelint": "^13.8.0",
"stylelint-config-prettier": "^8.0.2",
"stylelint-config-standard": "^20.0.0",

View file

@ -1,5 +1,6 @@
const redis = require('redis');
const { promisify } = require('util');
const level = require('level');
const path = require('path');
const fs = require('fs');
let hasWarned = false;
@ -10,28 +11,53 @@ const getConversion = {
stream: (x) => Buffer.from(x),
};
class RedisKVShim {
constructor(redisClient, namespace) {
class KVShim {
constructor(namespace) {
this.namespace = namespace;
this._redis = redisClient;
this._redisGet = promisify(this._redis.get).bind(redisClient);
this._redisSetex = promisify(this._redis.setex).bind(redisClient);
this._redisSet = promisify(this._redis.set).bind(redisClient);
this._redisDel = promisify(this._redis.del).bind(redisClient);
fs.mkdirSync(path.resolve(__dirname, '../../.devdbs'), {
recursive: true,
});
(async () => {
this.level = await level(path.resolve(__dirname, '../../.devdbs', namespace));
})();
}
key(key) {
return `${this.namespace}__${key}`;
makeValue(value, expirationTtl) {
if (!expirationTtl) {
return {
value,
expires: false,
};
}
return {
value: JSON.stringify(value),
expires: Date.now() + 1000 * expirationTtl,
};
}
validate(value) {
if (!value) {
return false;
}
if (value.expires < Date.now()) {
return false;
}
return true;
}
async get(key, type = 'text') {
const result = await this._redisGet(this.key(key));
const result = await this.level.get(key);
if (!result) {
if (!this.validate(result)) {
return null;
}
return getConversion[type](result);
return getConversion[type](result.value);
}
async getWithMetadata(key, type) {
@ -49,15 +75,11 @@ class RedisKVShim {
hasWarned = true;
}
if (expirationTtl) {
return this._redisSetex(this.key(key), value, expirationTtl);
}
return this._redisSet(this.key(key), value);
return this.level.put(key, this.makeValue(value, expirationTtl));
}
async delete(key) {
return this._redisDel(this.key(key));
return this.level.del(key);
}
list() {
@ -71,5 +93,5 @@ class RedisKVShim {
}
module.exports = {
RedisKVShim,
KVShim,
};

View file

@ -7,22 +7,15 @@ const chokidar = require('chokidar');
const webpack = require('webpack');
const { Crypto } = require('@peculiar/webcrypto');
const roleypolyConfig = require('../backend-worker/roleypoly.config');
const redis = require('redis');
const { RedisKVShim } = require('./kv');
const { KVShim } = require('./kv');
const crypto = new Crypto();
const redisClient = redis.createClient({ host: 'redis' });
const getKVs = (redisClient, namespaces = []) => {
namespaces.reduce(
(acc, ns) => ({ ...acc, [ns]: new RedisKVShim(redisClient, ns) }),
{}
);
};
const getKVs = (namespaces = []) =>
namespaces.reduce((acc, ns) => ({ ...acc, [ns]: new KVShim(ns) }), {});
const workerShims = {
...roleypolyConfig.environment,
...getKVs(redisClient, roleypolyConfig.kv),
...getKVs(roleypolyConfig.kv),
};
let listeners = [];

180
yarn.lock
View file

@ -3195,6 +3195,28 @@ abort-controller@3.0.0:
dependencies:
event-target-shim "^5.0.0"
abstract-leveldown@^6.2.1:
version "6.3.0"
resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a"
integrity sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ==
dependencies:
buffer "^5.5.0"
immediate "^3.2.3"
level-concat-iterator "~2.0.0"
level-supports "~1.0.0"
xtend "~4.0.0"
abstract-leveldown@~6.2.1, abstract-leveldown@~6.2.3:
version "6.2.3"
resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz#036543d87e3710f2528e47040bc3261b77a9a8eb"
integrity sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==
dependencies:
buffer "^5.5.0"
immediate "^3.2.3"
level-concat-iterator "~2.0.0"
level-supports "~1.0.0"
xtend "~4.0.0"
accepts@~1.3.7:
version "1.3.7"
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd"
@ -4397,7 +4419,7 @@ buffer@^4.3.0:
ieee754 "^1.1.4"
isarray "^1.0.0"
buffer@^5.5.0:
buffer@^5.5.0, buffer@^5.6.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
@ -5524,6 +5546,14 @@ deepmerge@^4.2.2:
resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
deferred-leveldown@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz#27a997ad95408b61161aa69bd489b86c71b78058"
integrity sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==
dependencies:
abstract-leveldown "~6.2.1"
inherits "^2.0.3"
define-properties@^1.1.2, define-properties@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1"
@ -5568,11 +5598,6 @@ delegates@^1.0.0:
resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
denque@^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/denque/-/denque-1.4.1.tgz#6744ff7641c148c3f8a69c307e51235c1f4a37cf"
integrity sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==
depd@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
@ -5984,6 +6009,16 @@ encodeurl@~1.0.2:
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
encoding-down@^6.3.0:
version "6.3.0"
resolved "https://registry.yarnpkg.com/encoding-down/-/encoding-down-6.3.0.tgz#b1c4eb0e1728c146ecaef8e32963c549e76d082b"
integrity sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==
dependencies:
abstract-leveldown "^6.2.1"
inherits "^2.0.3"
level-codec "^9.0.0"
level-errors "^2.0.0"
end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1:
version "1.4.4"
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
@ -6091,7 +6126,7 @@ enzyme@^3.11.0:
rst-selector-parser "^2.2.3"
string.prototype.trim "^1.2.1"
errno@^0.1.3, errno@~0.1.7:
errno@^0.1.3, errno@~0.1.1, errno@~0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618"
integrity sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==
@ -7696,6 +7731,11 @@ ignore@^5.1.4, ignore@^5.1.8:
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
immediate@^3.2.3:
version "3.3.0"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.3.0.tgz#1aef225517836bcdf7f2a2de2600c79ff0269266"
integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==
immer@1.10.0:
version "1.10.0"
resolved "https://registry.yarnpkg.com/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d"
@ -9128,6 +9168,88 @@ left-pad@^1.3.0:
resolved "https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e"
integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==
level-codec@^9.0.0:
version "9.0.2"
resolved "https://registry.yarnpkg.com/level-codec/-/level-codec-9.0.2.tgz#fd60df8c64786a80d44e63423096ffead63d8cbc"
integrity sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==
dependencies:
buffer "^5.6.0"
level-concat-iterator@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz#1d1009cf108340252cb38c51f9727311193e6263"
integrity sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==
level-errors@^2.0.0, level-errors@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/level-errors/-/level-errors-2.0.1.tgz#2132a677bf4e679ce029f517c2f17432800c05c8"
integrity sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==
dependencies:
errno "~0.1.1"
level-iterator-stream@~4.0.0:
version "4.0.2"
resolved "https://registry.yarnpkg.com/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz#7ceba69b713b0d7e22fcc0d1f128ccdc8a24f79c"
integrity sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==
dependencies:
inherits "^2.0.4"
readable-stream "^3.4.0"
xtend "^4.0.2"
level-js@^5.0.0:
version "5.0.2"
resolved "https://registry.yarnpkg.com/level-js/-/level-js-5.0.2.tgz#5e280b8f93abd9ef3a305b13faf0b5397c969b55"
integrity sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==
dependencies:
abstract-leveldown "~6.2.3"
buffer "^5.5.0"
inherits "^2.0.3"
ltgt "^2.1.2"
level-packager@^5.1.0:
version "5.1.1"
resolved "https://registry.yarnpkg.com/level-packager/-/level-packager-5.1.1.tgz#323ec842d6babe7336f70299c14df2e329c18939"
integrity sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==
dependencies:
encoding-down "^6.3.0"
levelup "^4.3.2"
level-supports@~1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-1.0.1.tgz#2f530a596834c7301622521988e2c36bb77d122d"
integrity sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==
dependencies:
xtend "^4.0.2"
level@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/level/-/level-6.0.1.tgz#dc34c5edb81846a6de5079eac15706334b0d7cd6"
integrity sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==
dependencies:
level-js "^5.0.0"
level-packager "^5.1.0"
leveldown "^5.4.0"
leveldown@^5.4.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-5.6.0.tgz#16ba937bb2991c6094e13ac5a6898ee66d3eee98"
integrity sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==
dependencies:
abstract-leveldown "~6.2.1"
napi-macros "~2.0.0"
node-gyp-build "~4.1.0"
levelup@^4.3.2:
version "4.4.0"
resolved "https://registry.yarnpkg.com/levelup/-/levelup-4.4.0.tgz#f89da3a228c38deb49c48f88a70fb71f01cafed6"
integrity sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==
dependencies:
deferred-leveldown "~5.3.0"
level-errors "~2.0.0"
level-iterator-stream "~4.0.0"
level-supports "~1.0.0"
xtend "~4.0.0"
leven@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2"
@ -9391,6 +9513,11 @@ lru-cache@^5.1.1:
dependencies:
yallist "^3.0.2"
ltgt@^2.1.2:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5"
integrity sha1-81ypHEk/e3PaDgdJUwTxezH4fuU=
make-dir@^2.0.0, make-dir@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
@ -9903,6 +10030,11 @@ napi-build-utils@^1.0.1:
resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806"
integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==
napi-macros@~2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b"
integrity sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==
native-url@0.3.4:
version "0.3.4"
resolved "https://registry.yarnpkg.com/native-url/-/native-url-0.3.4.tgz#29c943172aed86c63cee62c8c04db7f5756661f8"
@ -10047,6 +10179,11 @@ node-fetch@2.6.1, node-fetch@^2.6.0, node-fetch@^2.6.1:
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
node-gyp-build@~4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb"
integrity sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==
node-html-parser@1.4.9:
version "1.4.9"
resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-1.4.9.tgz#3c8f6cac46479fae5800725edb532e9ae8fd816c"
@ -11729,33 +11866,6 @@ redent@^3.0.0:
indent-string "^4.0.0"
strip-indent "^3.0.0"
redis-commands@^1.5.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.6.0.tgz#36d4ca42ae9ed29815cdb30ad9f97982eba1ce23"
integrity sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==
redis-errors@^1.0.0, redis-errors@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
redis-parser@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
dependencies:
redis-errors "^1.0.0"
redis@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a"
integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==
dependencies:
denque "^1.4.1"
redis-commands "^1.5.0"
redis-errors "^1.2.0"
redis-parser "^3.0.0"
reflect.ownkeys@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/reflect.ownkeys/-/reflect.ownkeys-0.2.0.tgz#749aceec7f3fdf8b63f927a04809e90c5c0b3460"
@ -14443,7 +14553,7 @@ xmlchars@^2.2.0:
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1:
xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==