Auto saved by Logseq
This commit is contained in:
Matthias Eckert
parent
df36dbdf30
commit
e36daa7bd6
@ -219,6 +219,131 @@
|
||||
tmp_assigning_customer_ids s
|
||||
WHERE s.uuid = p.uuid;
|
||||
DROP TABLE IF EXISTS tmp_assigning_customer_ids;
|
||||
|
||||
|
||||
|
||||
|
||||
copy tmp_dabe_isfrom (id, url, site_id, feed_ids) from '/nfs/export/PagesIsFrom/product_pages_categories_final_test.csv' delimiter E'\t' HEADER csv;
|
||||
|
||||
CREATE TABLE "tmp_dabe_isfrom" (
|
||||
"id" INTEGER NULL DEFAULT NULL,
|
||||
"url" TEXT NULL DEFAULT NULL,
|
||||
"site_id" INTEGER NULL DEFAULT NULL,
|
||||
"feed_ids" INTEGER NULL DEFAULT NULL
|
||||
)
|
||||
;
|
||||
|
||||
create index on tmp_dabe_isfrom using BTREE (url);
|
||||
|
||||
SELECT url, array_agg(feed_ids) FROM tmp_dabe_isfrom t GROUP BY url LIMIT 10
|
||||
|
||||
SELECT url, array_agg(feed_ids) FROM tmp_dabe_isfrom t WHERE url = 'https://www.amazon.cn/dp/B00DAEYLQW/' GROUP BY url
|
||||
|
||||
CREATE TABLE tmp_dabe_isfrom_agg AS
|
||||
SELECT url, array_agg(feed_ids) FROM tmp_dabe_isfrom t GROUP BY url;
|
||||
create index on tmp_dabe_isfrom_agg using BTREE (url);
|
||||
|
||||
ALTER TABLE tmp_dabe_isfrom_agg OWNER TO productstore;
|
||||
|
||||
|
||||
UPDATE bkcp_ent_productpage bp SET NAME=pp.name FROM ent_productpage pp WHERE bp.uuid=pp.uuid AND bp.name IS null
|
||||
|
||||
create table if not exists bckp_ent_productpage_parallel (
|
||||
uuid uuid,
|
||||
creation_date timestamptz,
|
||||
modification_date timestamptz,
|
||||
creator_id text,
|
||||
last_modifier_id text,
|
||||
name text,
|
||||
url text,
|
||||
url_normalized text,
|
||||
former_urls text[],
|
||||
crawl_date timestamptz,
|
||||
language bigint,
|
||||
country bigint,
|
||||
breadcrumb text,
|
||||
brand text[],
|
||||
image_url text,
|
||||
current_price numeric,
|
||||
number_of_replies bigint,
|
||||
number_of_reviews bigint,
|
||||
asin text[],
|
||||
asin_normalized text[],
|
||||
gtin text[],
|
||||
gtin_normalized text[],
|
||||
ean13 text[],
|
||||
ean13_normalized text[],
|
||||
sku text[],
|
||||
sku_normalized text[],
|
||||
vib text[],
|
||||
vib_normalized text[],
|
||||
modelnumber text,
|
||||
modelnumber_normalized text,
|
||||
shopcategory_id bigint[],
|
||||
shop_id bigint,
|
||||
shop_internal_id text,
|
||||
meta_info jsonb,
|
||||
booking_customer_ids hstore,
|
||||
assigned_cp_ids hstore,
|
||||
assigning_customer_ids hstore,
|
||||
ignoring_customer_ids hstore,
|
||||
CONSTRAINT _304d3bd1_13a8_4f02_842f_d011f92625fc UNIQUE(uuid),
|
||||
PRIMARY KEY (uuid)) PARTITION BY HASH (uuid);
|
||||
create table bckp_ent_productpage_parallel_0 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 0);
|
||||
create table bckp_ent_productpage_parallel_1 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 1);
|
||||
create table bckp_ent_productpage_parallel_2 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 2);
|
||||
create table bckp_ent_productpage_parallel_3 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 3);
|
||||
create table bckp_ent_productpage_parallel_4 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 4);
|
||||
create table bckp_ent_productpage_parallel_5 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 5);
|
||||
create table bckp_ent_productpage_parallel_6 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 6);
|
||||
create table bckp_ent_productpage_parallel_7 partition of bckp_ent_productpage_parallel FOR VALUES WITH (MODULUS 8, REMAINDER 7);
|
||||
|
||||
INSERT INTO bckp_ent_productpage_parallel SELECT * FROM bkcp_ent_productpage;
|
||||
|
||||
create index on bckp_ent_productpage_parallel using BTREE (url);
|
||||
create index on bckp_ent_productpage_parallel using GIN (former_urls);
|
||||
|
||||
|
||||
UPDATE bckp_ent_productpage_parallel pp SET shopcategory_id=array_agg FROM tmp_dabe_isfrom_agg di WHERE di.url=pp.url OR di.url=ANY(pp.former_urls);
|
||||
|
||||
|
||||
productstore=# UPDATE bckp_ent_productpage_parallel pp SET shopcategory_id=array_agg FROM tmp_dabe_isfrom_agg di WHERE di.url=pp.url OR di.url=ANY(pp.former_urls);
|
||||
|
||||
UPDATE 8856484
|
||||
Time: 700867.954 ms (11:40.868)
|
||||
|
||||
|
||||
|
||||
TRUNCATE rel_isfrom;
|
||||
INSERT INTO rel_isfrom (uuid, creation_date, modification_date, source_uuid, target_uuid) SELECT uuidv3(a.uuid||'isFrom'||b.uuid),NOW(),NOW(), a.uuid, b.uuid FROM bckp_ent_productpage_parallel a inner join ent_shopcategory b on b.dataretrieval_id = any(a.shopcategory_id);
|
||||
INSERT 0 21718375
|
||||
Time: 384528.971 ms (06:24.529)
|
||||
|
||||
|
||||
CREATE INDEX ON ent_productpage USING GIN (booking_customer_ids);
|
||||
|
||||
create index on ent_productpage using gin (name gin_trgm_ops);
|
||||
|
||||
CREATE INDEX ON ent_productpage USING GIN (ignoring_customer_ids);
|
||||
CREATE INDEX ON ent_productpage USING GIN (assigned_cp_ids);
|
||||
CREATE INDEX ON ent_productpage USING GIN (assigning_customer_ids);
|
||||
|
||||
UPDATE ent_customerproduct SET pid=c.pid, cid=c.cid
|
||||
FROM ent_customer c
|
||||
WHERE ent_customerproduct.ucid=c.ucid AND (ent_customerproduct.pid IS NULL OR ent_customerproduct.cid IS NULL);
|
||||
|
||||
|
||||
|
||||
CREATE TABLE tmp_ent_customerproduct_fixed_meta (like ent_customerproduct)
|
||||
|
||||
COPY tmp_ent_customerproduct_fixed_meta(uuid,creation_date,modification_date,creator_id,last_modifier_id,brand,asin,asin_normalized,ean13,ean13_normalized,sku,sku_normalized,name,language,vib,vib_normalized,pid,cid,ucid,bigLabelUrls,import_date,eigenbrand,meta_info) FROM '/nfs/export/spark_nach_maintenance/CustomerProduct/customer_product.merged.json.csv' WITH DELIMITER '|' QUOTE '^' HEADER CSV;
|
||||
|
||||
COPY tmp_ent_customerproduct_fixed_meta(uuid,creation_date,modification_date,creator_id,last_modifier_id,brand,asin,asin_normalized,ean13,ean13_normalized,sku,sku_normalized,name,language,vib,vib_normalized,pid,cid,ucid,bigLabelUrls,import_date,eigenbrand,meta_info) FROM '/nfs/export/spark_nach_maintenance/CustomerProduct/customer_product.merged.jsondatefix.csv' WITH DELIMITER '|' QUOTE '^' HEADER CSV;
|
||||
|
||||
SELECT cp.uuid AS CP_UUID, cp.name AS CP_NAME, p.url AS PAGE_URL, p.uuid AS PAGE_UUID, array_length(akeys(p.assigned_cp_ids), 1) AS c FROM ent_customerproduct cp, ent_productpage p WHERE cp.uuid = ANY(akeys(p.assigned_cp_ids)::uuid[]) GROUP BY cp.uuid, p.uuid, p.url HAVING array_length(akeys(p.assigned_cp_ids), 1) > 1
|
||||
|
||||
COPY (SELECT cp.uuid AS CP_UUID, cp.name AS CP_NAME, p.url AS PAGE_URL, p.uuid AS PAGE_UUID, array_length(akeys(p.assigned_cp_ids), 1) AS c FROM ent_customerproduct cp, ent_productpage p WHERE cp.uuid = ANY(akeys(p.assigned_cp_ids)::uuid[]) GROUP BY cp.uuid, p.uuid, p.url HAVING array_length(akeys(p.assigned_cp_ids), 1) > 1) to '/nfs/export/bsh_assigned_pages_more_than_one_cp.csv' WITH CSV;
|
||||
|
||||
```
|
||||
- #+BEGIN_QUOTE
|
||||
Save translations
|
||||
@ -3023,4 +3148,49 @@
|
||||
|
||||
station-dev-001.vicosrv.de /srv/sql/station-dev1_va.sql
|
||||
|
||||
```
|
||||
- ```
|
||||
virtualbox ubuntu
|
||||
|
||||
Host
|
||||
|
||||
NatNetwork in Virtualbox mit Port Weiterleitung
|
||||
Host IP 127.0.0.1 Port 22
|
||||
Gast IP 10.0.2.4 Port 22
|
||||
|
||||
In .ssh/config
|
||||
|
||||
Host ubunutu localhost
|
||||
User eckertms
|
||||
Hostname localhost
|
||||
|
||||
|
||||
Gast
|
||||
|
||||
sudo apt install openssh server
|
||||
ssh-keygen -t rsa
|
||||
|
||||
add host:.ssh/id_rsa.pub to guest:.ssh/authorized_keys
|
||||
|
||||
|
||||
Install Docker
|
||||
|
||||
https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-18-04
|
||||
|
||||
sudo apt upgrade
|
||||
sudo apt install apt-transport-https ca-certificates curl software-properties-common
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
|
||||
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
|
||||
sudo apt update
|
||||
apt-cache policy docker-ce
|
||||
sudo apt install docker-ce
|
||||
sudo systemctl status docker
|
||||
|
||||
sudo usermod -aG docker eckertms
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
```
|
||||
Loading…
x
Reference in New Issue
Block a user