Quicker db init

Signed-off-by: fly <merspieler@airmail.cc>
This commit is contained in:
fly 2021-09-30 09:15:32 +02:00
parent 05892dd003
commit f8e67c3cea
2 changed files with 332 additions and 114 deletions

View file

@ -22,7 +22,9 @@ from common import norm, get_tile_width, get_tile, get_area_name
dbuser= "" dbuser= ""
dbpw = "" dbpw = ""
dbname = "worldbuild" dbname = "worldbuild"
dbhost = "localhost"
force = False force = False
use_sqlfile = True
argc = len(sys.argv) argc = len(sys.argv)
i = 1 i = 1
@ -36,8 +38,13 @@ while i < argc:
elif sys.argv[i] == "-d" or sys.argv[i] == "--database": elif sys.argv[i] == "-d" or sys.argv[i] == "--database":
i += 1 i += 1
dbname = sys.argv[i] dbname = sys.argv[i]
elif sys.argv[i] == "-h" or sys.argv[i] == "--host":
i += 1
dbhost = sys.argv[i]
elif sys.argv[i] == "-f" or sys.argv[i] == "--force": elif sys.argv[i] == "-f" or sys.argv[i] == "--force":
force = True force = True
elif sys.argv[i] == "-N" or sys.argv[i] == "--no-sql":
force = True
elif sys.argv[i] == "-h" or sys.argv[i] == "--help": elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
print("usage: init-db.py [OPTIONS]") print("usage: init-db.py [OPTIONS]")
print("Initializes worldbuild status db") print("Initializes worldbuild status db")
@ -46,7 +53,10 @@ while i < argc:
print(" -d, --database Database to be initialised. Default: worldbuild") print(" -d, --database Database to be initialised. Default: worldbuild")
print(" -p, --password Database password") print(" -p, --password Database password")
print(" -u, --user Database user") print(" -u, --user Database user")
print(" -h, --host Database host")
print(" -f, --force Recreates everything from scratch") print(" -f, --force Recreates everything from scratch")
print(" -N, --no-sql Don't use the sql file and create the structure from scratch")
print(" Only recommended when underlaying tiling structure was changed")
print(" -h, --help Shows this help and exit") print(" -h, --help Shows this help and exit")
sys.exit(0) sys.exit(0)
else: else:
@ -62,21 +72,22 @@ if dbpw == "":
print("ERROR: No database password given") print("ERROR: No database password given")
sys.exit(1) sys.exit(1)
states = [ #try:
if use_sqlfile:
else:
states = [
"pending", "pending",
"started", "started",
"done", "done",
"packaged", "packaged",
"skip", "skip",
"rebuild" "rebuild"
] ]
db = pymysql.connect(dbhost, dbuser, dbpw, dbname)
#try: cursor = db.cursor()
db = pymysql.connect("localhost", dbuser, dbpw, dbname)
cursor = db.cursor() if force:
if force:
cursor.execute("DROP TABLE IF EXISTS tile") cursor.execute("DROP TABLE IF EXISTS tile")
cursor.execute("DROP TABLE IF EXISTS secondLevel") cursor.execute("DROP TABLE IF EXISTS secondLevel")
cursor.execute("DROP TABLE IF EXISTS topLevel") cursor.execute("DROP TABLE IF EXISTS topLevel")
@ -84,26 +95,26 @@ if force:
cursor.execute("DROP TRIGGER IF EXISTS update_top_level") cursor.execute("DROP TRIGGER IF EXISTS update_top_level")
cursor.execute("DROP TRIGGER IF EXISTS update_second_level") cursor.execute("DROP TRIGGER IF EXISTS update_second_level")
cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL, color VARCHAR(20));") cursor.execute("CREATE TABLE IF NOT EXISTS status (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(15) UNIQUE NOT NULL, color VARCHAR(20));")
cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));") cursor.execute("CREATE TABLE IF NOT EXISTS topLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id));")
cursor.execute("CREATE TABLE IF NOT EXISTS secondLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, parent_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id), FOREIGN KEY (parent_id) REFERENCES topLevel(id));") cursor.execute("CREATE TABLE IF NOT EXISTS secondLevel (id INT PRIMARY KEY AUTO_INCREMENT, name CHAR(7) UNIQUE NOT NULL, status_id INT NOT NULL, parent_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id), FOREIGN KEY (parent_id) REFERENCES topLevel(id));")
cursor.execute("CREATE TABLE IF NOT EXISTS tile (id INT PRIMARY KEY AUTO_INCREMENT, status_id INT NOT NULL, parent_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id), FOREIGN KEY (parent_id) REFERENCES secondLevel(id));") cursor.execute("CREATE TABLE IF NOT EXISTS tile (id INT PRIMARY KEY AUTO_INCREMENT, status_id INT NOT NULL, parent_id INT NOT NULL, FOREIGN KEY (status_id) REFERENCES status(id), FOREIGN KEY (parent_id) REFERENCES secondLevel(id));")
cursor.execute("CREATE INDEX IF NOT EXISTS tile_index on tile(status_id, parent_id);") cursor.execute("CREATE INDEX IF NOT EXISTS tile_index on tile(status_id, parent_id);")
for state in states: for state in states:
sql = "INSERT INTO status(name) VALUES ('" + state + "')" sql = "INSERT INTO status(name) VALUES ('" + state + "')"
cursor.execute(sql) cursor.execute(sql)
sql = "SELECT id FROM status WHERE name='pending'" sql = "SELECT id FROM status WHERE name='pending'"
cursor.execute(sql) cursor.execute(sql)
result = cursor.fetchall() result = cursor.fetchall()
for row in result: for row in result:
sid = str(row[0]) sid = str(row[0])
print("Creating tile groups") print("Creating tile groups")
ii = -9 ii = -9
while ii < 9: while ii < 9:
i = ii * 10 i = ii * 10
jj = -18 jj = -18
while jj < 18: while jj < 18:
@ -130,9 +141,9 @@ while ii < 9:
jj += 1 jj += 1
ii += 1 ii += 1
print("Creating tiles") print("Creating tiles")
lat = -83 lat = -83
while lat < 83: while lat < 83:
lon = -180 lon = -180
while lon < 180: while lon < 180:
tile = get_tile(lat, lon) tile = get_tile(lat, lon)
@ -147,60 +158,31 @@ while lat < 83:
lon += get_tile_width(lat) lon += get_tile_width(lat)
lat += 0.125 lat += 0.125
db.commit() sql = "CREATE TABLE `auth` (`id` int(11) NOT NULL AUTO_INCREMENT, `token` varchar(100) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `token` (`token`)) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4"
cursor.execute(sql)
# Get states db.commit()
#SELECT status.name AS state, COUNT(status.name) AS occ FROM tile INNER JOIN status ON tile.status_id = status.id WHERE tile.parent_id = 400 GROUP BY status.name;
# Always create triggers via script cause there can be a user name conflict on import.
print("Creating triggers") print("Creating triggers")
# Trigger for updating secondLevel on tile update # Trigger for updating secondLevel on tile update
sql = ('CREATE OR REPLACE TRIGGER update_second_level ' sql = ('CREATE OR REPLACE TRIGGER update_second_level '
'AFTER UPDATE ON tile ' 'AFTER UPDATE ON tile '
'FOR EACH ROW ' 'FOR EACH ROW '
'BEGIN ' 'BEGIN UPDATE secondLevel SET status_id = (SELECT status_id FROM tile JOIN status ON status_id = status.id WHERE tile.parent_id = NEW.parent_id ORDER BY priority ASC LIMIT 1) WHERE id = NEW.parent_id; END')
'IF (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id) = (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "packaged")) THEN '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "packaged") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "started")) > 0 THEN '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "started") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "rebuild")) > 0 THEN '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "rebuild") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "skip")) > 0 THEN '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "skip") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM tile WHERE tile.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "done")) > 0 THEN '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "done") WHERE id = NEW.parent_id; '
'ELSE '
'UPDATE secondLevel SET status_id = (SELECT id FROM status WHERE name = "pending") WHERE id = NEW.parent_id; '
'END IF; '
'END; ')
cursor.execute(sql) cursor.execute(sql)
# Trigger for updating topLevel on secondLevel update # Trigger for updating topLevel on secondLevel update
sql = ('CREATE OR REPLACE TRIGGER update_top_level ' sql = ('CREATE OR REPLACE TRIGGER update_top_level '
'AFTER UPDATE ON secondLevel ' 'AFTER UPDATE ON secondLevel '
'FOR EACH ROW ' 'FOR EACH ROW '
'BEGIN ' 'BEGIN UPDATE topLevel SET status_id = (SELECT status_id FROM secondLevel JOIN status ON status_id = status.id WHERE secondLevel.parent_id = NEW.parent_id ORDER BY priority ASC LIMIT 1) WHERE id = NEW.parent_id; END')
'IF (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id) = (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "packaged")) THEN '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "packaged") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "started")) > 0 THEN '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "started") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "rebuild")) > 0 THEN '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "rebuild") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "skip")) > 0 THEN '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "skip") WHERE id = NEW.parent_id; '
'ELSEIF (SELECT COUNT(status_id) FROM secondLevel WHERE secondLevel.parent_id = NEW.parent_id AND status_id = (SELECT id FROM status WHERE name = "done")) > 0 THEN '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "done") WHERE id = NEW.parent_id; '
'ELSE '
'UPDATE topLevel SET status_id = (SELECT id FROM status WHERE name = "pending") WHERE id = NEW.parent_id; '
'END IF; '
'END; ')
cursor.execute(sql)
sql = "CREATE TABLE `auth` (`id` int(11) NOT NULL AUTO_INCREMENT, `token` varchar(100) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `token` (`token`)) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4"
cursor.execute(sql) cursor.execute(sql)
db.commit() db.commit()
#except: #except:
# db.rollback()
# print("Failed to setup database...") # print("Failed to setup database...")
# db.rollback()
#finally: #finally:
db.close() db.close()

236
worldbuild-base.sql Normal file

File diff suppressed because one or more lines are too long