o pfE@sddlZddlZddlZddlZddlmZddlZddlZddl m Z ddl m Z ddl mZddlmZdZdZd Zgd ZGd d d ZdS) N)Iterator)ConnectionPool)Metric)Tile)Tilesetmetadatagenerate_stats tile_stats) gg?g?g?g?gffffff?gGz?g+?g?c @seZdZdZd?defddZ d@dd Z d ed ed ed eddf ddZ d eddfddZ de e fddZ de efddZ de efddZd@ddZ ddZdefddZddZd d!Z d ed"e efd#d$ZdAd efd%d&Zd ed'edeedBejdBffd(d)Z *dBd ed'ed+edejdBfd,d-Zd.d/Zd0d1Zd2d3Z d@d4d5Z!d6d7Z"d8d9Z#d ed:eddfd;d<Z$d ed'efd=d>Z%dS)CStoragez Storage is an object representing a tile storage, backed by a PostgreSQL database A Storage contains tiles and metadata about tilesets, and has functions to update tiles and metadata based on the ID tilekilndbpoolcCs||_||_dSN)_Storage__pool_Storage__schema)selfr schemarI/home/ertert/spirit/venv/lib/python3.10/site-packages/tilekiln/storage.py__init__ s zStorage.__init__returnNc Cs|j=}| }|d|jd|||||Wdn1s-wYWddSWddS1sEwYdS)NzCREATE SCHEMA IF NOT EXISTS "")r connectioncursorexecuter_Storage__setup_stats_Storage__setup_metadatacommit)rconncurrrr create_schema's     "zStorage.create_schemaidminzoommaxzoomtilejsonc Cs|j:}|}||||||||||||Wdn1s*wYWddSWddS1sBwYdSr )rrr_Storage__set_metadata_Storage__setup_tablesrrr r!r"r#rrrrrcreate_tileset4s   "zStorage.create_tilesetc Cs|jT}|7}|d|jdtd|f|d|jd|d|d|jdtd|f|Wdn1sDwYWddSWddS1s\wYdS)N DELETE FROM ""."z" WHERE id = %sz DROP TABLE "z " CASCADE)rrrrrMETADATA_TABLETILE_STATS_TABLEr)rr rrrrrremove_tileset=s   "zStorage.remove_tilesetc cs|jR}d|_|jtjjd.}|d|jdt d|D]}t ||d|d|dt |d Vq"Wd n1sCwYWd d SWd d S1s[wYd S) z2 Gets all tilesets in the storage T row_factoryzISELECT id, minzoom, maxzoom, tilejson FROM "r)rr r!r"r#N) rr read_onlyrpsycopgrowsdict_rowrrr*rjsondumpsrrrrecordrrr get_tilesetsGs$   "zStorage.get_tilesetsc cs|jC}d|_|jtjjd}|d|jdt d|D]}|dVq"Wdn1s4wYWddSWddS1sLwYdS)z* Get only the tileset IDs Tr-z-SELECT id FROM "r)rr N) rrr/rr0r1r2rrr*r5rrrget_tileset_idsTs    "zStorage.get_tileset_idsc cs|jC}d|_|jtjtd}|d|j dt d|D]}|Vq$Wdn1s4wYWddSWddS1sLwYdS)NTr-zQSELECT id, zoom, num_tiles, size, percentiles FROM "r)r) rrr/rr0r1 class_rowrrrr+r5rrrmetricsas   "zStorage.metricsc Cs|jB}|%}|D]}||}||}|||||q|Wdn1s2wYWddSWddS1sJwYdSr )rrrr8 get_minzoom get_maxzoom _Storage__update_tileset_metricsr)rrrr r!r"rrrupdate_metricsjs      "zStorage.update_metricsc Cs|j2}|}|||||||Wdn1s"wYWddSWddS1s:wYdS)zn Saves metadata into storage This just wraps __set_metadata, which requires a cursor N)rrrr$rr&rrr set_metadatats   "zStorage.set_metadatac Cs|jc}d|_|jtjjdG}|d|jdt d|f| }|dur9t j d|ddd t d |d }|d g|d <t|WdWdS1s[wYWddS1skwYdS)z+Gets the tilejson for a layer from storage.Tr-z6SELECT tilejson FROM "r)/" WHERE id = %sNz#Failed to retrieve tilejson for id , does it exist in storage DB?errr#z/{z}/{x}/{y}.mvttiles)rrr/rr0r1r2rrr*fetchoneclickechosysexitr3r4)rr urlrrresultr#rrr get_tilejsons.  "zStorage.get_tilejsonc C|jV}d|_|jtjjd:}|d|jdt d|f| }|dur9t j d|ddd t d |d WdWdS1sNwYWddS1s^wYdS) *Gets the minzoom for a layer from storage.Tr-z5SELECT minzoom FROM "r)r@N"Failed to retrieve minzoom for id rArBrDr!rrr/rr0r1r2rrr*rFrGrHrIrJrr rrrLrrrr;*  "zStorage.get_minzoomc CrN) rOTr-z5SELECT maxzoom FROM "r)r@NrPrArBrDr"rQrRrrrr<rSzStorage.get_maxzoomrEc Csx|j-}|}|D] }||||q Wdn1s!wY|WddS1s5wYdSr )rrr_Storage__delete_tiler)rr rErrtilerrr delete_tiless   "zStorage.delete_tilesc Cs|jF}|)}|durt||||d}|D] }||||q|Wdn1s6wYWddSWddS1sNwYdS)NrD)rrrranger;r<_Storage__truncate_tabler)rr zoomsrrzoomrrrtruncate_tabless   "zStorage.truncate_tablesrUc Cs|je}|d|jtjjdG}|jd|jd|d|j|j |j fdd| }|durA WdWddSt |d |d fWdWdS1s]wYWddS1smwYdS) NSET TIMEZONE TO 'GMT'r-zSELECT generated, tile FROM "r)zG" WHERE zoom = %s AND x = %s AND y = %sT)binary)NNrU generated)rrrrr0r1r2rrZxyrFgzip decompress)rr rUrrrLrrrget_tiles    "zStorage.get_tilertiledatac Cs|jj}|d|jtjjdL}|d|j}|d|jd|d|j|j |j t j |ddf| }|durM WdWddS|d WdWdS1sbwYWddS1srwYdS) Nr\r-_z INSERT INTO "r)z" AS store (zoom, x, y, tile) VALUES (%s, %s, %s, %s) ON CONFLICT (zoom, x, y) DO UPDATE SET tile = EXCLUDED.tile, generated = CASE WHEN store.tile != EXCLUDED.tile THEN statement_timestamp() ELSE store.generated END RETURNING generatedr)mtimer^)rrrrr0r1r2rZrr_r`racompressrF) rr rUrd render_timerr tablenamerLrrr save_tiles"     "zStorage.save_tilecCs|d|jdtddS)zE Create the metadata table in storage. This is safe to rerun CREATE TABLE IF NOT EXISTS "r)z" ( id text PRIMARY KEY, active boolean NOT NULL DEFAULT TRUE, minzoom smallint NOT NULL, maxzoom smallint NOT NULL, tilejson jsonb NOT NULL)Nrrr*rrrrr__setup_metadataszStorage.__setup_metadatacCs(|d|jdtd||||fdS)z Sets metadata using a cursor This is separate from set_metadata because sometimes it needs calling within a transaction rfr)z" (id, minzoom, maxzoom, tilejson) VALUES (%s, %s, %s, %s) ON CONFLICT (id) DO UPDATE SET minzoom = EXCLUDED.minzoom, maxzoom = EXCLUDED.maxzoom, tilejson = EXCLUDED.tilejson Nrm)rrr r!r"r#rrr__set_metadatas zStorage.__set_metadatacCs8|d|jdtd|d|jdtddS)zqCreate the stats tables. One table has tile generation stats, the other has tile storage stats. z1CREATE UNLOGGED TABLE IF NOT EXISTS "r)z" ( id text, zoom smallint, num_rendered integer DEFAULT 0, time_rendered interval DEFAULT '0', PRIMARY KEY (id, zoom) ) rla " ( id text, zoom smallint, num_tiles integer NOT NULL, size bigint NOT NULL, percentiles double precision[][] NOT NULL, PRIMARY KEY (id, zoom), CHECK (array_length(percentiles, 1) = 2) ) N)rrGENERATE_STATS_TABLEr+rnrrr __setup_stats s    zStorage.__setup_statsc Csbt||dD]'}|d|}|d|d|jdtd|jd|d ||tdqdS) NrDrezSET LOCAL jit TO ON;rfr)ay" SELECT %(id)s AS id, %(zoom)s AS zoom, COUNT(*) AS num_tiles, COALESCE(SUM(length(tile)),0) AS size, ARRAY[%(percentile)s, COALESCE(PERCENTILE_CONT(%(percentile)s::double precision[]) WITHIN GROUP (ORDER BY length(tile)), array_fill(0, ARRAY[array_length(%(percentile)s, 1)]))] AS percentiles FROM "a" ON CONFLICT (id, zoom) DO UPDATE SET num_tiles = EXCLUDED.num_tiles, size = EXCLUDED.size, percentiles = EXCLUDED.percentiles; )r rZ percentile)rWrrr+ PERCENTILESrrr r!r"rZrjrrr__update_tileset_metrics-s    z Storage.__update_tileset_metricscCs|d|jd|d|d|d t||dD]-}|d|}|d|jd|d|jd|d |d |d |jd|d qd S)zCreate the tile storage tables This creates the tile storage tables. It intentionally does not try to overwrite existing tables. zCREATE TABLE "r)z5" ( zoom smallint CHECK (zoom >= z AND zoom <= aH), x int CHECK (x >= 0 AND x < 1 << zoom), y int CHECK (y >= 0 AND y < 1 << zoom), generated timestamptz DEFAULT statement_timestamp(), tile bytea NOT NULL, primary key (zoom, x, y) ) PARTITION BY LIST (zoom)rDrez," PARTITION OF "z-" FOR VALUES IN ()z ALTER TABLE "zD" ALTER COLUMN tile SET STORAGE EXTERNALN)rrrWrurrr__setup_tablesOs   zStorage.__setup_tablesc Cs|ja}|jtjjd@}|d|jdtd|j f| }|dur8t j d|j ddd t d |d |_|d |_|d |_Wdn1sQwYWddSWddS1siwYdS)znLoad the stored metadata. This allows serving a TileJSON without having access to the config r-zHSELECT minzoom, maxzoom, tilejson FROM "r)r@Nz#Failed to retrieve metadata for id rATrBrDr!r"r#)rrrr0r1r2rrr*r rFrGrHrIrJr!r"_Storage__rawtilejson)rrrrLrrr__load_metadatags,     "zStorage.__load_metadatarZcCs,|d|}|d|jd|ddS)z4Remove every tile from a particular tileset and zoomrezTRUNCATE TABLE "r)rN)rr)rrr rZrjrrr__truncate_table|szStorage.__truncate_tablecCs,|d|jd|d|j|j|jfdS)a?Delete an individual tile How this is implemented is not ideal for long lists of tiles to delete, but generally a long list is an entire zoom or a box. In the former case it is implemented as __truncate_table, and the latter case is not implemented but would take min/max x/y. r(r)z?" WHERE zoom = %s AND x = %s AND y = %sN)rrrZr_r`)rrr rUrrr __delete_tiles zStorage.__delete_tile)r )rNr )r)&__name__ __module__ __qualname____doc__rrrstrintr'r,rrr7r8rr:r>r?rMr;r<rrVr[tuplebytesdatetimercrkrr$rr=r%_Storage__load_metadatarXrTrrrrr sH         (     !"r )rrar3rIcollections.abcrrG psycopg.rowsr0 psycopg_poolrtilekiln.metricr tilekiln.tilertilekiln.tilesetrr*rqr+rtr rrrrs