Add serverside Firebase code, add new features for saving presets

This commit is contained in:
Logan Williams
2023-07-26 10:53:18 +02:00
parent 5576b4ae23
commit 0e96107036
11 changed files with 418 additions and 128 deletions

3
.gitignore vendored
View File

@@ -22,3 +22,6 @@ pnpm-debug.log*
*.njsproj
*.sln
*.sw?
firebaseConfig.js

View File

@@ -7,8 +7,6 @@ name = "pypi"
flask = "*"
psycopg2 = "*"
flask-cors = "*"
google-oauth = "*"
google-auth = "*"
gunicorn = "*"
loguru = "*"

View File

@@ -4,15 +4,20 @@ from psycopg2.extras import RealDictCursor
from flask import Flask, request, jsonify, abort, Response
from flask_cors import CORS
import json
from google.oauth2 import id_token
import google.auth.transport
from functools import wraps
import os
from loguru import logger
from datetime import datetime
import math
import firebase_admin
from firebase_admin import credentials
from firebase_admin import auth
from firebase_admin import firestore
import time
GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID", None)
cred = credentials.Certificate("service_account.json")
firebase_app = firebase_admin.initialize_app(cred)
db = firestore.client()
app = Flask(__name__)
app.config["TEMPLATES_AUTO_RELOAD"] = True
@@ -20,39 +25,39 @@ app.config["TEMPLATES_AUTO_RELOAD"] = True
CORS(app)
ALLOWED_COMPARISONS = [
'=',
'!=',
'>',
'<',
'>=',
'<=',
'starts with',
'ends with',
'contains',
'does not contain',
'is null',
'is not null',
]
ALLOWED_METHODS = [
"OR",
"AND"
"=",
"!=",
">",
"<",
">=",
"<=",
"starts with",
"ends with",
"contains",
"does not contain",
"is null",
"is not null",
]
ALLOWED_CASTS = [
"integer",
"float",
"cast_to_int",
"cast_to_float"
]
ALLOWED_METHODS = ["OR", "AND"]
ALLOWED_CASTS = ["integer", "float", "cast_to_int", "cast_to_float"]
def get_db_connection():
conn = psycopg2.connect(database=os.environ.get("PG_DB"), host=os.environ.get("PG_HOST"), port=os.environ.get("PG_PORT"), user=os.environ.get("PG_USER"), password=os.environ.get("PG_PASSWORD"))
conn = psycopg2.connect(
database=os.environ.get("PG_DB"),
host=os.environ.get("PG_HOST"),
port=os.environ.get("PG_PORT"),
user=os.environ.get("PG_USER"),
password=os.environ.get("PG_PASSWORD"),
)
return conn
def json_query(query, conn=None):
def query_with_timing(query, conn=None):
if conn is None:
conn = get_db_connection()
conn = get_db_connection()
cur = conn.cursor(cursor_factory=RealDictCursor)
@@ -71,8 +76,23 @@ def json_query(query, conn=None):
t2 = datetime.now()
logger.info(f"Found {len(data)} results in {t2 - t1} seconds")
return jsonify(data)
for d in data:
d.pop('point_geom')
return (data, t2 - t1)
def get_user(request):
token = None
if "Authorization" in request.headers:
token = request.headers["Authorization"].split(" ")[1]
if not token:
return None
idinfo = auth.verify_id_token(token)
return idinfo
def token_required(f):
@wraps(f)
@@ -86,24 +106,24 @@ def token_required(f):
return {
"message": "Authentication Token is missing!",
"data": None,
"error": "Unauthorized"
"error": "Unauthorized",
}, 401
try:
idinfo = id_token.verify_oauth2_token(token, google.auth.transport.requests.Request(), GOOGLE_CLIENT_ID)
idinfo = auth.verify_id_token(token)
if idinfo is None:
logger.warning(f"Invalid authentication token {token}")
return {
"message": "Invalid Authentication token!",
"data": None,
"error": "Unauthorized"
}, 403
"message": "Invalid Authentication token!",
"data": None,
"error": "Unauthorized",
}, 403
except Exception as e:
logger.warning(f"Other error {e}")
return {
"message": "Something went wrong",
"data": None,
"error": str(e)
"error": str(e),
}, 403
logger.info(f"Authenticated request by {idinfo['email']}")
@@ -111,59 +131,87 @@ def token_required(f):
return decorated
def make_filter_query(filter):
filter_query = sql.SQL("")
i = 0
for subfilter in filter['filters']:
if subfilter['comparison'] not in ALLOWED_COMPARISONS:
for subfilter in filter["filters"]:
if subfilter["comparison"] not in ALLOWED_COMPARISONS:
logger.error(f"Invalid comparison {subfilter['comparison']}")
break
if subfilter['comparison'] == '=':
filter_query = sql.SQL("{filter_query} (tags @> {match})").format(filter_query=filter_query, match=sql.Literal(subfilter['parameter'] + '=>' + subfilter['value']))
elif subfilter['comparison'] == '!=':
filter_query = sql.SQL("{filter_query} NOT(tags @> {match})").format(filter_query=filter_query, match=sql.Literal(subfilter['parameter'] + '=>' + subfilter['value']))
elif subfilter['comparison'] == 'is null':
filter_query = sql.SQL("{filter_query} NOT(tags?{parameter})").format(filter_query=filter_query, parameter=sql.Literal(subfilter['parameter']))
elif subfilter['comparison'] == 'is not null':
filter_query = sql.SQL("{filter_query} (tags?{parameter})").format(filter_query=filter_query, parameter=sql.Literal(subfilter['parameter']))
if subfilter["comparison"] == "=":
filter_query = sql.SQL("{filter_query} (tags @> {match})").format(
filter_query=filter_query,
match=sql.Literal(subfilter["parameter"] + "=>" + subfilter["value"]),
)
elif subfilter["comparison"] == "!=":
filter_query = sql.SQL("{filter_query} NOT(tags @> {match})").format(
filter_query=filter_query,
match=sql.Literal(subfilter["parameter"] + "=>" + subfilter["value"]),
)
elif subfilter["comparison"] == "is null":
filter_query = sql.SQL("{filter_query} NOT(tags?{parameter})").format(
filter_query=filter_query, parameter=sql.Literal(subfilter["parameter"])
)
elif subfilter["comparison"] == "is not null":
filter_query = sql.SQL("{filter_query} (tags?{parameter})").format(
filter_query=filter_query, parameter=sql.Literal(subfilter["parameter"])
)
else:
parameter = sql.SQL("tags->{parameter}").format(parameter=sql.Literal(subfilter['parameter']))
parameter = sql.SQL("tags->{parameter}").format(
parameter=sql.Literal(subfilter["parameter"])
)
if 'cast' in subfilter and subfilter['cast'] in ALLOWED_CASTS:
if subfilter['cast'] == 'cast_to_float':
parameter = sql.SQL("cast_to_float({parameter}, 0.0)").format(parameter=parameter)
elif subfilter['cast'] == 'cast_to_int':
parameter = sql.SQL("cast_to_int({parameter}, 0)").format(parameter=parameter)
if "cast" in subfilter and subfilter["cast"] in ALLOWED_CASTS:
if subfilter["cast"] == "cast_to_float":
parameter = sql.SQL("cast_to_float({parameter}, 0.0)").format(
parameter=parameter
)
elif subfilter["cast"] == "cast_to_int":
parameter = sql.SQL("cast_to_int({parameter}, 0)").format(
parameter=parameter
)
else:
parameter = sql.SQL("CAST({parameter} AS {cast})").format(parameter=parameter, cast=sql.SQL(subfilter['cast']))
parameter = sql.SQL("CAST({parameter} AS {cast})").format(
parameter=parameter, cast=sql.SQL(subfilter["cast"])
)
if subfilter['comparison'] == 'starts with':
subfilter['value'] = f"{subfilter['value']}%"
subfilter['comparison'] = 'ILIKE'
elif subfilter['comparison'] == 'ends with':
subfilter['value'] = f"%{subfilter['value']}"
subfilter['comparison'] = 'ILIKE'
elif subfilter['comparison'] == 'contains':
subfilter['value'] = f"%{subfilter['value']}%"
subfilter['comparison'] = 'ILIKE'
elif subfilter['comparison'] == 'does not contain':
subfilter['value'] = f"%{subfilter['value']}%"
subfilter['comparison'] = 'NOT ILIKE'
if subfilter["comparison"] == "starts with":
subfilter["value"] = f"{subfilter['value']}%"
subfilter["comparison"] = "ILIKE"
elif subfilter["comparison"] == "ends with":
subfilter["value"] = f"%{subfilter['value']}"
subfilter["comparison"] = "ILIKE"
elif subfilter["comparison"] == "contains":
subfilter["value"] = f"%{subfilter['value']}%"
subfilter["comparison"] = "ILIKE"
elif subfilter["comparison"] == "does not contain":
subfilter["value"] = f"%{subfilter['value']}%"
subfilter["comparison"] = "NOT ILIKE"
filter_query = sql.SQL("{filter_query} ({parameter} {comparison} {value})").format(filter_query=filter_query, parameter=parameter, comparison=sql.SQL(subfilter['comparison']), value=sql.Literal(subfilter['value']))
filter_query = sql.SQL(
"{filter_query} ({parameter} {comparison} {value})"
).format(
filter_query=filter_query,
parameter=parameter,
comparison=sql.SQL(subfilter["comparison"]),
value=sql.Literal(subfilter["value"]),
)
if i != len(filter['filters']) - 1:
filter_query = sql.SQL("{filter_query} {method}").format(filter_query=filter_query, method=sql.SQL(filter['method']))
if i != len(filter["filters"]) - 1:
filter_query = sql.SQL("{filter_query} {method}").format(
filter_query=filter_query, method=sql.SQL(filter["method"])
)
i += 1
return filter_query
@app.route('/intersection')
@app.route("/intersection")
@token_required
def get_intersection():
args = request.args
@@ -178,23 +226,44 @@ def get_intersection():
bbox = [l, b, r, t]
area = math.pow(6371,2) * math.pi * abs(math.sin(math.radians(t)) - math.sin(math.radians(b))) * abs(r - l) / 180
area = (
math.pow(6371, 2)
* math.pi
* abs(math.sin(math.radians(t)) - math.sin(math.radians(b)))
* abs(r - l)
/ 180
)
# reject queries that are too large
if area > 4e6:
return Response(status=400)
bbox_filter = sql.SQL("AND (way && ST_Transform(ST_MakeEnvelope({left}, {bottom}, {right}, {top}, 4326), 3857))").format(left=sql.Literal(bbox[0]), bottom=sql.Literal(bbox[1]), right=sql.Literal(bbox[2]), top=sql.Literal(bbox[3]))
bbox_filter = sql.SQL(
"AND (way && ST_Transform(ST_MakeEnvelope({left}, {bottom}, {right}, {top}, 4326), 3857))"
).format(
left=sql.Literal(bbox[0]),
bottom=sql.Literal(bbox[1]),
right=sql.Literal(bbox[2]),
top=sql.Literal(bbox[3]),
)
first = filters[0]
first_query = sql.SQL("SELECT tags->'name' AS name, ST_Centroid(way) AS point_geom, way AS geom FROM {table}").format(table=
sql.SQL('planet_osm_line') if first['type'] == 'line' else
sql.SQL('planet_osm_polygon') if first['type'] == 'polygon' else
sql.SQL('planet_osm_point') if first['type'] == 'point' else
sql.SQL('planet_osm'))
first_query = sql.SQL(
"SELECT tags->'name' AS name, ST_Centroid(way) AS point_geom, way AS geom FROM {table}"
).format(
table=sql.SQL("planet_osm_line")
if first["type"] == "line"
else sql.SQL("planet_osm_polygon")
if first["type"] == "polygon"
else sql.SQL("planet_osm_point")
if first["type"] == "point"
else sql.SQL("planet_osm")
)
first_filter = make_filter_query(first)
first_assembled = sql.SQL("{query} WHERE ({filter}) {bbox}").format(query=first_query, filter=first_filter, bbox=bbox_filter)
first_assembled = sql.SQL("{query} WHERE ({filter}) {bbox}").format(
query=first_query, filter=first_filter, bbox=bbox_filter
)
logger.info(f"Buffer: {buffer}\tFilters: {filters}\tBbox: [{l},{b},{r},{t}]")
@@ -202,40 +271,74 @@ def get_intersection():
for f in filters[1:]:
filter = make_filter_query(f)
if f['type'] == 'point':
if f["type"] == "point":
query = sql.SQL("SELECT way AS geom FROM planet_osm_point")
elif f['type'] == 'line':
elif f["type"] == "line":
query = sql.SQL("SELECT way AS geom FROM planet_osm_line")
elif f['type'] == 'polygon':
elif f["type"] == "polygon":
query = sql.SQL("SELECT way AS geom FROM planet_osm_polygon")
elif f['type'] == 'any':
elif f["type"] == "any":
query = sql.SQL("SELECT way AS geom FROM planet_osm")
assembled = sql.SQL("{query} WHERE ({filter}) {bbox}").format(query=query, filter=filter, bbox=bbox_filter)
assembled = sql.SQL("{query} WHERE ({filter}) {bbox}").format(
query=query, filter=filter, bbox=bbox_filter
)
subqueries.append(assembled)
join_query = sql.SQL("SELECT DISTINCT point_geom, name, ST_Y(ST_Transform(point_geom, 4326)) AS lat, ST_X(ST_Transform(point_geom, 4326)) as lng FROM ({point}) point ").format(point=first_assembled)
join_query = sql.SQL(
"SELECT DISTINCT point_geom, name, ST_Y(ST_Transform(point_geom, 4326)) AS lat, ST_X(ST_Transform(point_geom, 4326)) as lng FROM ({point}) point "
).format(point=first_assembled)
i = 0
for q in subqueries:
join_query = sql.SQL("{join_query} JOIN ({q}) {subindex} ON ST_DWithin(point.geom, {subindex}.geom, {buffer})").format(join_query=join_query, q=q, subindex=sql.SQL('subquery' + str(i)), buffer=sql.Literal(buffer))
join_query = sql.SQL(
"{join_query} JOIN ({q}) {subindex} ON ST_DWithin(point.geom, {subindex}.geom, {buffer})"
).format(
join_query=join_query,
q=q,
subindex=sql.SQL("subquery" + str(i)),
buffer=sql.Literal(buffer),
)
i += 1
join_query = sql.SQL("{join_query} LIMIT 100").format(join_query=join_query)
conn = get_db_connection()
logger.info(f"Executing query: {join_query.as_string(conn)}")
return json_query(join_query)
timestamp = time.time_ns()
user = get_user(request)
@app.route('/robots.txt')
data, tdiff = query_with_timing(join_query)
log_data = {
"timestamp": firestore.SERVER_TIMESTAMP,
"query": join_query.as_string(conn),
"user_uid": user["uid"],
"user_email": user["email"],
"bbox": bbox,
"filters": filters,
"query_time": tdiff.total_seconds(),
"query_nresults": len(data),
"query_results": data,
}
db.collection("searches").document(str(timestamp)).set(log_data)
logger.info(f"Found {len(data)} results in {tdiff} seconds")
return jsonify(data)
@app.route("/robots.txt")
def robots():
return Response("User-agent: *\nDisallow: /", mimetype='text/plain')
return Response("User-agent: *\nDisallow: /", mimetype="text/plain")
def start():
app.run(port=5050)
if __name__ == '__main__':
if __name__ == "__main__":
start()

View File

@@ -114,6 +114,7 @@ export default {
type: this.selectedQueryType,
filters: this.filters.filter((v) => v.parameter != ""),
method: this.method,
unsavedCustomFeature: true,
},
]);

View File

@@ -30,24 +30,76 @@
<v-card>
<v-card-title>Feature presets</v-card-title>
<v-card-text>
<v-chip
v-for="query in queries"
:key="query.name + query.type"
:color="
query.type == 'point'
? '#8BC34A'
: query.type == 'line'
? '#46d4db'
: query.type == 'polygon'
? '#FFC107'
: '#BEBEBE'
"
draggable
@dragstart="startDrag($event, query)"
style="margin: 0.25em"
@click="addFeature(query)"
>{{ query.name }}</v-chip
>
<div>
<v-chip
v-for="query in $store.state.presets"
:key="query.name + query.type"
:color="
query.type == 'point'
? '#8BC34A'
: query.type == 'line'
? '#46d4db'
: query.type == 'polygon'
? '#FFC107'
: '#BEBEBE'
"
draggable
@dragstart="startDrag($event, query)"
style="margin: 0.25em"
@click="addFeature(query)"
>{{ query.name }}</v-chip
>
</div>
<div v-if="$store.state.customPresets.length > 0">
<span class="custom-header">Custom presets</span>
<v-chip
v-for="query in $store.state.customPresets"
:key="query.id"
:color="
query.type == 'point'
? '#8BC34A'
: query.type == 'line'
? '#46d4db'
: query.type == 'polygon'
? '#FFC107'
: '#BEBEBE'
"
draggable
@dragstart="startDrag($event, query)"
style="margin: 0.25em"
@click="addFeature(query)"
close
@click:close="deleteDialog = query"
>{{ query.name }}</v-chip
>
</div>
<v-dialog :value="deleteDialog" width="auto">
<v-card>
<v-card-title>Delete custom preset</v-card-title>
<v-card-text>
Are you sure you want to delete the custom preset "{{
deleteDialog.name
}}" from your account?
</v-card-text>
<v-card-actions style="padding-bottom: 1em; margin-top: -0.5em">
<v-btn
color="red"
@click="
removePreset(deleteDialog);
deleteDialog = false;
"
>Delete</v-btn
>
<v-btn
color="primary"
style="margin-right: 0em; margin-left: auto"
@click="deleteDialog = false"
>Keep preset</v-btn
>
</v-card-actions>
</v-card>
</v-dialog>
</v-card-text>
</v-card>
<FeatureCustom />
@@ -56,7 +108,6 @@
</template>
<script>
import queries from "./queries.js";
import FeatureView from "./FeatureView.vue";
import FeatureCustom from "./FeatureCustom.vue";
@@ -68,8 +119,8 @@ export default {
},
data() {
return {
queries,
accepting: false,
deleteDialog: false,
};
},
methods: {
@@ -94,6 +145,9 @@ export default {
startDrag(e, item) {
e.dataTransfer.setData("object", JSON.stringify(item));
},
removePreset(f) {
this.$store.dispatch("removePreset", f.id);
},
},
};
</script>
@@ -106,4 +160,11 @@ export default {
.type {
font-style: italic;
}
.custom-header {
font-weight: bold;
color: black;
margin-left: 0.5em;
margin-right: 0.5em;
}
</style>

View File

@@ -26,7 +26,45 @@
</v-card-text>
<v-card-actions>
<v-btn color="red" text @click="remove(index)"> Remove </v-btn>
<v-btn
color="blue"
v-if="query.unsavedCustomFeature"
text
@click="dialog = true"
>
Save feature preset
</v-btn>
</v-card-actions>
<v-dialog v-model="dialog" activator="parent" width="auto">
<v-card>
<v-card-title>Save this feature as a preset</v-card-title>
<v-card-text>
Your saved presets are only visible to you.
<v-text-field
label="Preset name"
required
v-model="name"
:error="error"
></v-text-field>
</v-card-text>
<v-card-actions style="padding-bottom: 1em; margin-top: -0.5em">
<v-btn
color="red"
@click="
dialog = false;
error = false;
"
>Cancel</v-btn
>
<v-btn
color="primary"
style="margin-right: 0em; margin-left: auto"
@click="tryToSave(index)"
>Save preset</v-btn
>
</v-card-actions>
</v-card>
</v-dialog>
</v-card>
</template>
@@ -37,10 +75,37 @@ export default {
query: Object,
index: Number,
},
data() {
return {
dialog: false,
name: "",
error: false,
};
},
methods: {
remove(index) {
this.$store.commit("removeSelected", index);
},
tryToSave(index) {
if (this.name.length == 0) {
this.error = true;
return;
}
let oldSelected = this.$store.state.selected;
oldSelected[index].unsavedCustomFeature = false;
oldSelected[index].name = this.name;
this.$store.commit("updateSelected", oldSelected);
this.dialog = false;
this.$store.dispatch("savePreset", { index, name: this.name });
},
},
watch: {
name(newName) {
if (newName.length > 0) {
this.error = false;
}
},
},
};
</script>

View File

@@ -28,6 +28,13 @@ export default {
firebase.auth().onAuthStateChanged((user) => {
this.$store.commit("setUser", user);
if (user) {
user.getIdToken().then((token) => {
this.$store.commit("setToken", token);
});
this.$store.dispatch("getCustomPresets");
}
});
ui.start("#firebaseui-auth-container", uiConfig);

View File

@@ -3,7 +3,7 @@
<v-row style="padding: 0.75em">
<HelpCard />
</v-row>
<feature-selector />
<FeatureSelector />
<v-alert
type="error"
style="padding: 0.75em; margin-top: 1em"

View File

@@ -1,15 +1,7 @@
import { initializeApp } from "firebase/app";
import { getAuth } from "firebase/auth";
import { getFirestore } from "firebase/firestore";
const firebaseConfig = {
apiKey: "AIzaSyBN5oJ8c_VGhcfesAxXPVmuVnJ_V5MM8JM",
authDomain: "osm-search-364115.firebaseapp.com",
projectId: "osm-search-364115",
storageBucket: "osm-search-364115.appspot.com",
messagingSenderId: "919009657823",
appId: "1:919009657823:web:f3be7f8470a6c36665ba6a",
};
import { firebaseConfig } from "./firebaseConfig.js";
const firebaseApp = initializeApp(firebaseConfig);

View File

@@ -2,6 +2,18 @@ import Vue from "vue";
import Vuex from "vuex";
import firebase from "firebase/compat/app";
import "firebase/compat/auth";
import { firebaseFirestore } from "@/firebase";
import {
collection,
addDoc,
serverTimestamp,
query,
where,
getDocs,
deleteDoc,
doc,
} from "firebase/firestore";
import queries from "@/assets/queries";
Vue.use(Vuex);
@@ -25,11 +37,16 @@ export default new Vuex.Store({
osmKeys: [],
selectedKeyValues: [],
user: null,
presets: queries,
customPresets: [],
},
mutations: {
setUser(state, user) {
state.user = user;
},
setToken(state, token) {
state.token = token;
},
updateSelected(state, value) {
state.selected = [...value];
},
@@ -84,6 +101,9 @@ export default new Vuex.Store({
setSelectedKeyValues(state, values) {
state.selectedKeyValues = values;
},
setCustomPresets(state, presets) {
state.customPresets = presets;
},
},
actions: {
async signout({ commit }) {
@@ -93,6 +113,7 @@ export default new Vuex.Store({
// clean user from store
commit("setUser", null);
commit("setToken", null);
} catch (error) {
console.error("signOutUser (firebase/auth.js): ", error);
}
@@ -188,6 +209,7 @@ export default new Vuex.Store({
}
});
},
searchLocation({ commit }, search_text) {
fetch(
`https://api.mapbox.com/geocoding/v5/mapbox.places/${search_text}.json?access_token=${process.env.VUE_APP_MAPBOX_TOKEN}`
@@ -216,6 +238,44 @@ export default new Vuex.Store({
});
});
},
async savePreset({ state, dispatch }, { index, name }) {
try {
const docRef = await addDoc(collection(firebaseFirestore, "presets"), {
filters: state.selected[index].filters,
method: state.selected[index].method,
type: state.selected[index].type,
name: name,
author_uid: state.user.uid,
timestamp: serverTimestamp(),
});
console.log("Document written with ID: ", docRef.id);
dispatch("getCustomPresets");
} catch (e) {
console.error("Error adding document: ", e);
}
},
async getCustomPresets({ state, commit }) {
const q = query(
collection(firebaseFirestore, "presets"),
where("author_uid", "==", state.user.uid)
);
const querySnapshot = await getDocs(q);
const customPresets = querySnapshot.docs.map((d) => ({
...d.data(),
id: d.id,
}));
commit("setCustomPresets", customPresets);
},
async removePreset({ dispatch }, id) {
await deleteDoc(doc(firebaseFirestore, "presets", id));
dispatch("getCustomPresets");
},
},
modules: {},
});