2025-summer/team-3
12
Fork 0
Code Issues Pull requests Projects Releases Activity

remove packages, hosters, ... completely

Browse source
This commit is contained in:
Laura Klünder 2017-05-01 18:10:46 +02:00
parent f0cec9b7bf
commit 5efb6d537d
37 changed files with 49 additions and 1603 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/c3nav/access/apply.py
View file

@ -1,42 +1,23 @@
from django.conf import settings
from django.db.models import Q
from c3nav.mapdata.inclusion import get_maybe_invisible_areas_names
from c3nav.mapdata.utils.cache import get_packages_cached
def get_public_packages():
packages_cached = get_packages_cached()
return [packages_cached[name] for name in settings.PUBLIC_PACKAGES]
def get_nonpublic_packages():
packages_cached = get_packages_cached()
return [package for name, package in packages_cached.items() if name not in settings.PUBLIC_PACKAGES]
def get_unlocked_packages(request, packages_cached=None):
return tuple(get_packages_cached().values()) if request.c3nav_full_access else get_public_packages()
def get_unlocked_packages_names(request, packages_cached=None):
if request.c3nav_full_access:
return get_packages_cached().keys()
return settings.PUBLIC_PACKAGES
def can_access_package(request, package):
return request.c3nav_full_access or package.name in get_unlocked_packages_names(request)
def can_access(request, item):
# todo implement this
return True
def filter_queryset_by_access(request, queryset, filter_location_inclusion=False):
return queryset if request.c3nav_full_access else queryset.filter(package__in=get_public_packages())
# todo implement this
return queryset if request.c3nav_full_access else queryset.filter(public=True)
def filter_arealocations_by_access(request, queryset):
# todo implement this
if request.c3nav_full_access:
return queryset
return queryset.filter(Q(Q(package__in=get_public_packages()), ~Q(routing_inclusion='needs_permission')) |
return queryset.filter(Q(Q(public=True), ~Q(routing_inclusion='needs_permission')) |
Q(name__in=request.c3nav_access_list))

53
src/c3nav/access/templates/access/prove.html
View file

@ -1,53 +0,0 @@
{% extends 'access/base.html' %}
{% load bootstrap3 %}
{% load i18n %}
{% block bodyclass %}login{% endblock %}
{% block content %}
<form method="POST">
{% csrf_token %}
<fieldset>
<legend>{% trans 'Prove access rights' %}</legend>
<p>{% blocktrans %}Please enter a valid authentication code for the hosters of the following non-public map packages:{% endblocktrans %}</p>
{% if success %}
<div class="alert alert-success">
<strong>{% trans 'Thanks – you get full access to the map!' %}</strong><br>
{% if replaced %}{% trans 'All previous tokens have been invalidated.' %}<br>{% endif %}
</div>
{% include 'access/fragment_token.html' with token=token %}
{% elif hosters %}
{% if error %}
<div class="alert alert-dismissible alert-danger">
<button type="button" class="close" data-dismiss="alert">×</button>
{% if error == 'invalid' %}
<strong>{% trans 'Sorry.' %}</strong> {% trans 'One or more access tokens were not correct.' %}
{% elif error == 'duplicate' %}
<strong>{% trans 'Sorry.' %}</strong> {% trans 'You already have a valid access token.' %}
{% endif %}
</div>
{% endif %}
{% for package in hosters %}
<div class="form-group">
<label for="hoster{{ forloop.counter0 }}">{{ package.name }}</label>
<input type="password" class="form-control" id="hoster{{ forloop.counter0 }}" name="{{ package.name }}" placeholder="{% trans 'Access Token' %}">
</div>
{% endfor %}
<div class="checkbox">
<label>
<input type="checkbox" name="replace" value="1"> {% trans 'Invalidate previous token(s).' %}
</label>
</div>
<div class="form-group">
<button type="submit" class="btn btn-primary btn-block btn-lg">{% trans 'Submit' %}</button>
</div>
{% else %}
<div class="alert alert-info">
<strong>{% trans 'Sorry, this service is currently not available.' %}</strong>
</div>
{% endif %}
</fieldset>
</form>
{% endblock %}

3
src/c3nav/access/urls.py
View file

@ -1,11 +1,10 @@
from django.conf.urls import url
from django.contrib.auth import views as auth_views
from c3nav.access.views import activate_token, dashboard, prove, show_user_token, token_qr, user_detail, user_list
from c3nav.access.views import activate_token, dashboard, show_user_token, token_qr, user_detail, user_list
urlpatterns = [
url(r'^$', dashboard, name='access.dashboard'),
url(r'^prove/$', prove, name='access.prove'),
url(r'^activate/(?P<pk>[0-9]+):(?P<secret>[a-zA-Z0-9]+)/$', activate_token, name='access.activate'),
url(r'^qr/(?P<pk>[0-9]+):(?P<secret>[a-zA-Z0-9]+).png$', token_qr, name='access.qr'),
url(r'^users/$', user_list, name='access.users'),

59
src/c3nav/access/views.py
View file

@ -1,17 +1,12 @@
from collections import OrderedDict
import qrcode
from django.contrib.auth.decorators import login_required
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db import transaction
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from c3nav.access.apply import get_nonpublic_packages
from c3nav.access.forms import AccessTokenForm, AccessUserForm
from c3nav.access.models import AccessToken, AccessUser
from c3nav.editor.hosters import get_hoster_for_package
@login_required(login_url='/access/login/')
@ -19,60 +14,6 @@ def dashboard(request):
return redirect('access.users')
def prove(request):
hosters = OrderedDict((package, get_hoster_for_package(package)) for package in get_nonpublic_packages())
if not hosters or None in hosters.values():
return render(request, 'access/prove.html', context={'hosters': None})
error = None
if request.method == 'POST':
user_id = None
for package, hoster in hosters.items():
access_token = request.POST.get(package.name)
hoster_user_id = hoster.get_user_id_with_access_token(access_token)
if hoster_user_id is None:
return render(request, 'access/prove.html', context={
'hosters': hosters,
'error': 'invalid',
})
if user_id is None:
user_id = hoster_user_id
replaced = False
with transaction.atomic():
user = AccessUser.objects.filter(user_url=user_id).first()
if user is not None:
valid_tokens = user.valid_tokens
if valid_tokens.count():
if request.POST.get('replace') != '1':
return render(request, 'access/prove.html', context={
'hosters': hosters,
'error': 'duplicate',
})
for token in valid_tokens:
token.expired = True
token.save()
replaced = True
else:
user = AccessUser.objects.create(user_url=user_id)
token = user.new_token(permissions=':full', description='automatically created')
return render(request, 'access/prove.html', context={
'hosters': hosters,
'success': True,
'replaced': replaced,
'token': token,
})
return render(request, 'access/prove.html', context={
'hosters': hosters,
'error': error,
})
def activate_token(request, pk, secret):
token = get_object_or_404(AccessToken, expired=False, activated=False, id=pk, secret=secret)
if request.method == 'POST':

8
src/c3nav/api/urls.py
View file

@ -7,12 +7,9 @@ from rest_framework.generics import GenericAPIView
from rest_framework.response import Response
from rest_framework.routers import SimpleRouter
from c3nav.editor.api import HosterViewSet, SubmitTaskViewSet
from c3nav.mapdata.api import (GeometryTypeViewSet, GeometryViewSet, LevelViewSet, LocationViewSet, PackageViewSet,
SourceViewSet)
from c3nav.mapdata.api import (GeometryTypeViewSet, GeometryViewSet, LevelViewSet, LocationViewSet, SourceViewSet)
router = SimpleRouter()
router.register(r'packages', PackageViewSet)
router.register(r'levels', LevelViewSet)
router.register(r'sources', SourceViewSet)
@ -21,9 +18,6 @@ router.register(r'geometries', GeometryViewSet, base_name='geometry')
router.register(r'locations', LocationViewSet, base_name='location')
router.register(r'hosters', HosterViewSet, base_name='hoster')
router.register(r'submittasks', SubmitTaskViewSet, base_name='submittask')
class APIRoot(GenericAPIView):
"""

107
src/c3nav/editor/api.py
View file

@ -1,107 +0,0 @@
from collections import OrderedDict
from django.core import signing
from django.core.signing import BadSignature
from django.http import Http404
from rest_framework.decorators import detail_route
from rest_framework.exceptions import ValidationError
from rest_framework.response import Response
from rest_framework.viewsets import ViewSet
from c3nav.editor.hosters import get_hoster_for_package, hosters
from c3nav.editor.serializers import HosterSerializer, TaskSerializer
from c3nav.editor.tasks import submit_edit_task
from c3nav.mapdata.models.package import Package
class HosterViewSet(ViewSet):
"""
Retrieve and interact with package hosters
"""
lookup_field = 'name'
def retrieve(self, request, name=None):
if name not in hosters:
raise Http404
serializer = HosterSerializer(hosters[name], context={'request': request})
return Response(serializer.data)
@detail_route(methods=['get'])
def state(self, request, name=None):
if name not in hosters:
raise Http404
hoster = hosters[name]
state = hoster.get_state(request)
error = hoster.get_error(request) if state == 'logged_out' else None
return Response(OrderedDict((
('state', state),
('error', error),
)))
@detail_route(methods=['post'])
def auth_uri(self, request, name=None):
if name not in hosters:
raise Http404
return Response({
'auth_uri': hosters[name].get_auth_uri(request)
})
@detail_route(methods=['post'])
def submit(self, request, name=None):
if name not in hosters:
raise Http404
hoster = hosters[name]
if 'data' not in request.POST:
raise ValidationError('Missing POST parameter: data')
if 'commit_msg' not in request.POST:
raise ValidationError('Missing POST parameter: commit_msg')
data = request.POST['data']
commit_msg = request.POST['commit_msg'].strip()
if not commit_msg:
raise ValidationError('POST parameter may not be empty: commit_msg')
try:
data = signing.loads(data)
except BadSignature:
raise ValidationError('Bad data signature.')
if data['type'] != 'editor.edit':
raise ValidationError('Wrong data type.')
package = Package.objects.filter(name=data['package_name']).first()
data_hoster = None
if package is not None:
data_hoster = get_hoster_for_package(package)
if hoster != data_hoster:
raise ValidationError('Wrong hoster.')
data['commit_msg'] = commit_msg
task = hoster.submit_edit(request, data)
serializer = TaskSerializer(task, context={'request': request})
return Response(serializer.data)
class SubmitTaskViewSet(ViewSet):
"""
Get hoster submit tasks
"""
lookup_field = 'id_'
def retrieve(self, request, id_=None):
task = submit_edit_task.AsyncResult(task_id=id_)
try:
task.ready()
except:
raise Http404
serializer = TaskSerializer(task, context={'request': request})
return Response(serializer.data)

2
src/c3nav/editor/apps.py
View file

@ -5,7 +5,5 @@ class EditorConfig(AppConfig):
name = 'c3nav.editor'
def ready(self):
from c3nav.editor.hosters import init_hosters
from c3nav.editor.forms import create_editor_forms
init_hosters()
create_editor_forms()

24
src/c3nav/editor/forms.py
View file

@ -4,14 +4,10 @@ from collections import OrderedDict
from django.conf import settings
from django.forms import CharField, ModelForm, ValidationError
from django.forms.models import ModelChoiceField
from django.forms.widgets import HiddenInput
from django.utils.translation import ugettext_lazy as _
from shapely.geometry.geo import mapping
from c3nav.access.apply import get_unlocked_packages
from c3nav.mapdata.models import Package
class MapitemFormMixin(ModelForm):
def __init__(self, *args, request=None, **kwargs):
@ -26,24 +22,6 @@ class MapitemFormMixin(ModelForm):
if creating:
self.fields['name'].initial = hex(int(time.time()*1000000))[2:]
# restrict package choices and field_name
if not creating:
if not settings.DIRECT_EDITING:
self.fields['package'].widget = HiddenInput()
self.fields['package'].disabled = True
self.initial['package'] = self.instance.package.name
elif not settings.DIRECT_EDITING:
unlocked_packages = get_unlocked_packages(request)
if len(unlocked_packages) == 1:
self.fields['package'].widget = HiddenInput()
self.fields['package'].initial = next(iter(unlocked_packages))
self.fields['package'].disabled = True
else:
self.fields['package'] = ModelChoiceField(
queryset=Package.objects.filter(name__in=unlocked_packages),
)
self.fields['package'].to_field_name = 'name'
if 'level' in self.fields:
# hide level widget and set field_name
self.fields['level'].widget = HiddenInput()
@ -107,7 +85,7 @@ class MapitemFormMixin(ModelForm):
def create_editor_form(mapitemtype):
possible_fields = ['name', 'package', 'altitude', 'level', 'intermediate', 'levels', 'geometry', 'direction',
possible_fields = ['name', 'public', 'altitude', 'level', 'intermediate', 'levels', 'geometry', 'direction',
'elevator', 'button', 'crop_to_level', 'width', 'groups', 'override_altitude', 'color',
'location_type', 'can_search', 'can_describe', 'routing_inclusion', 'compiled_room', 'bssids']
existing_fields = [field.name for field in mapitemtype._meta.get_fields() if field.name in possible_fields]

29
src/c3nav/editor/hosters/__init__.py
View file

@ -1,29 +0,0 @@
from django.conf import settings
from c3nav.editor.hosters.github import GithubHoster # noqa
from c3nav.editor.hosters.gitlab import GitlabHoster # noqa
from collections import OrderedDict
hosters = {}
def init_hosters():
global hosters
hosters = OrderedDict((name, create_hoster(name=name, **data)) for name, data in settings.EDITOR_HOSTERS.items())
def create_hoster(api, **kwargs):
if api == 'github':
return GithubHoster(**kwargs)
elif api == 'gitlab':
return GitlabHoster(**kwargs)
else:
raise ValueError('Unknown hoster API: %s' % api)
def get_hoster_for_package(package):
for name, hoster in hosters.items():
if package.home_repo.startswith(hoster.base_url):
return hoster
return None

166
src/c3nav/editor/hosters/base.py
View file

@ -1,166 +0,0 @@
from abc import ABC, abstractmethod
from urllib.parse import urlparse, urlunparse
from celery.result import AsyncResult
from django.conf import settings
from django.urls.base import reverse
from django.utils.translation import ugettext_lazy as _
from c3nav.editor.tasks import check_access_token_task, request_access_token_task, submit_edit_task
from c3nav.mapdata.models import Package
class Hoster(ABC):
def __init__(self, name, base_url):
self.name = name
self.base_url = base_url
def get_packages(self):
"""
Get a Queryset of all packages that can be handled by this hoster
"""
return Package.objects.filter(home_repo__startswith=self.base_url).order_by('name')
def _get_callback_uri(self, request):
uri = request.build_absolute_uri(reverse('editor.oauth.callback', kwargs={'hoster': self.name}))
if settings.OAUTH_CALLBACK_SCHEME is None and settings.OAUTH_CALLBACK_NETLOC is None:
return uri
parts = list(urlparse(uri))
if settings.OAUTH_CALLBACK_SCHEME is not None:
parts[0] = settings.OAUTH_CALLBACK_SCHEME
if settings.OAUTH_CALLBACK_NETLOC is not None:
parts[1] = settings.OAUTH_CALLBACK_NETLOC
return urlunparse(parts)
def _get_session_data(self, request):
request.session.modified = True
return request.session.setdefault('hosters', {}).setdefault(self.name, {})
def get_error(self, request):
"""
If an error occured lately, return and forget it.
"""
session_data = self._get_session_data(request)
if 'error' in session_data:
return session_data.pop('error')
def get_state(self, request):
"""
Get current hoster state for this user.
:return: 'logged_in', 'logged_out', 'missing_permissions' or 'checking' if a check is currently running.
"""
session_data = self._get_session_data(request)
state = session_data.setdefault('state', 'logged_out')
if state == 'checking':
task = AsyncResult(id=session_data.get('checking_progress_id'))
if settings.CELERY_ALWAYS_EAGER:
task.maybe_reraise()
self._handle_checking_task(request, task, session_data)
state = session_data['state']
return state
def check_state(self, request):
"""
Sets the state for this user to 'checking' immediately and starts a task that checks if the currently known
is still valid and sets the state afterwards.
Does nothing if the current state is not 'logged_in'.
"""
session_data = self._get_session_data(request)
state = session_data.get('state')
if state == 'logged_in':
session_data['state'] = 'checking'
task = check_access_token_task.delay(hoster=self.name, access_token=session_data['access_token'])
if settings.CELERY_ALWAYS_EAGER:
task.maybe_reraise()
session_data['checking_progress_id'] = task.id
self._handle_checking_task(request, task, session_data)
def _handle_checking_task(self, request, task, session_data):
"""
Checks if the checking task is finished and if so handles its results.
"""
if task.ready():
if task.failed():
session_data['state'] = 'logged_out'
session_data['error'] = _('Internal error.')
else:
result = task.result
session_data.update(result) # updates 'state' key and optional 'error' and 'access_tokenÄ keys.
session_data.pop('checking_progress_id')
def request_access_token(self, request, *args, **kwargs):
"""
Starts a task that calls do_request_access_token.
"""
args = (self.name, )+args
session_data = self._get_session_data(request)
session_data['state'] = 'checking'
task = request_access_token_task.apply_async(args=args, kwargs=kwargs)
session_data['checking_progress_id'] = task.id
self._handle_checking_task(request, task, session_data)
def submit_edit(self, request, data):
session_data = self._get_session_data(request)
task = submit_edit_task.delay(hoster=self.name, access_token=session_data['access_token'], data=data)
if settings.CELERY_ALWAYS_EAGER:
task.maybe_reraise()
return task
@abstractmethod
def get_auth_uri(self, request):
"""
Get the a URL the user should be redirected to to authenticate and invalidates any previous URLs.
"""
pass
@abstractmethod
def handle_callback_request(self, request):
"""
Validates and handles the callback request and calls request_access_token.
"""
pass
@abstractmethod
def do_request_access_token(self, *args, **kwargs):
"""
Task method for requesting the access token asynchroniously.
Returns a dict with a 'state' key containing the new hoster state, an optional 'error' key containing an
error message and an optional 'access_token' key containing a new access token.
"""
pass
@abstractmethod
def do_check_access_token(self, access_token):
"""
Task method for checking the access token asynchroniously.
Returns a dict with a 'state' key containing the new hoster state.
"""
pass
def _submit_error(self, error):
return {
'success': False,
'error': error
}
@abstractmethod
def do_submit_edit(self, access_token, data):
"""
Task method for submitting an edit (e.g. creating a pull request).
Returns a dict with a 'success' key that contains a boolean, an optional 'error' key containing an error
message and an optional 'url' key containing an URL to the created pull request.
"""
pass
@abstractmethod
def get_user_id_with_access_token(self, access_token) -> str:
"""
Get User ID of the User with this access token or None if the access token does not work.
"""
pass

198
src/c3nav/editor/hosters/github.py
View file

@ -1,198 +0,0 @@
import base64
import string
import time
import uuid
from urllib.parse import urlencode
import requests
from django.core.exceptions import SuspiciousOperation
from django.utils.crypto import get_random_string
from c3nav.editor.hosters.base import Hoster
from c3nav.mapdata.models.package import Package
class GithubHoster(Hoster):
title = 'GitHub'
def __init__(self, app_id, app_secret, **kwargs):
super().__init__(**kwargs)
self._app_id = app_id
self._app_secret = app_secret
def get_auth_uri(self, request):
oauth_csrf_token = get_random_string(42, string.ascii_letters+string.digits)
self._get_session_data(request)['oauth_csrf_token'] = oauth_csrf_token
callback_uri = self._get_callback_uri(request).replace('://localhost:8000', 's://33c3.c3nav.de')
self._get_session_data(request)['callback_uri'] = callback_uri
return 'https://github.com/login/oauth/authorize?%s' % urlencode((
('client_id', self._app_id),
('redirect_uri', callback_uri),
('scope', 'public_repo'),
('state', oauth_csrf_token),
))
def handle_callback_request(self, request):
code = request.GET.get('code')
state = request.GET.get('state')
if code is None or state is None:
raise SuspiciousOperation('Missing parameters.')
session_data = self._get_session_data(request)
if session_data.get('oauth_csrf_token') != state:
raise SuspiciousOperation('OAuth CSRF token mismatch')
session_data.pop('oauth_csrf_token')
callback_uri = session_data.pop('callback_uri')
self.request_access_token(request, code, state, callback_uri)
def do_request_access_token(self, code, state, callback_uri):
response = requests.post('https://github.com/login/oauth/access_token', data={
'client_id': self._app_id,
'client_secret': self._app_secret,
'code': code,
'redirect_uri': callback_uri,
'state': state
}, headers={'Accept': 'application/json'}).json()
if 'error' in response:
return {
'state': 'logged_out',
'error': '%s: %s %s' % (response['error'], response['error_description'], response['error_uri'])
}
if 'public_repo' not in response['scope'].split(','):
return {
'state': 'missing_permissions',
'access_token': response['access_token']
}
return {
'state': 'logged_in',
'access_token': response['access_token']
}
def do_check_access_token(self, access_token):
response = requests.get('https://api.github.com/rate_limit', headers={'Authorization': 'token '+access_token})
if response.status_code != 200:
return {'state': 'logged_out'}
if 'public_repo' not in (s.strip() for s in response.headers.get('X-OAuth-Scopes').split(',')):
return {'state': 'missing_permissions'}
return {'state': 'logged_in'}
def do_submit_edit(self, access_token, data):
# Get endpoint URL with access token
def endpoint_url(endpoint):
return 'https://api.github.com/' + endpoint[1:] + '?access_token=' + access_token
# Check access token
state = self.do_check_access_token(access_token)['state']
if state == 'logged_out':
return self._submit_error('The access token is no longer working. Please sign in again.')
if state == 'missing_permissions':
return self._submit_error('Missing Permissions. Please sign in again.')
# Get Package from db
try:
package = Package.objects.get(name=data['package_name'])
except Package.DoesNotExist:
return self._submit_error('Could not find package.')
# Get repo name on this host, e.g. c3nav/c3nav
repo_name = '/'.join(package.home_repo[len(self.base_url):].split('/')[:2])
# todo: form
# Get user
response = requests.get(endpoint_url('/user'))
if response.status_code != 200:
return self._submit_error('Could not get user.')
user = response.json()
# Check if there is already a fork. If not, create one.
fork_name = user['login'] + '/' + repo_name.split('/')[1]
fork_created = False
for i in range(10):
response = requests.get(endpoint_url('/repos/%s' % fork_name), allow_redirects=False)
if response.status_code == 200:
# Something that could be a fork exists, check if it is one
fork = response.json()
if fork['fork'] and fork['parent']['full_name'] == repo_name:
# It's a fork and it's the right one!
break
else:
return self._submit_error('Could not create fork: there already is a repo with the same name.')
elif response.status_code in (404, 301):
if not fork_created:
# Fork does not exist, create it
# Creating forks happens asynchroniously, so we will stay in the loop to check repeatedly if the
# fork does exist until we run into a timeout.
response = requests.post(endpoint_url('/repos/%s/forks' % repo_name))
fork_created = True
else:
# Fork was not created yet. Wait a moment, then try again.
time.sleep(4)
else:
return self._submit_error('Could not check for existing fork: error %d' % response.status_code)
else:
# We checked multiple timeas and waited more than half a minute. Enough is enorugh.
return self._submit_error('Could not create fork: fork creation timeout.')
# Create branch
branch_name = 'editor-%s' % uuid.uuid4()
response = requests.post(endpoint_url('/repos/%s/git/refs' % fork_name),
json={'ref': 'refs/heads/'+branch_name, 'sha': data['commit_id']})
if response.status_code != 201:
return self._submit_error('Could not create branch.')
# Make commit
if data['action'] == 'create':
response = requests.put(endpoint_url('/repos/%s/contents/%s' % (fork_name, data['file_path'])),
json={'branch': branch_name, 'message': data['commit_msg'],
'content': base64.b64encode(data['content'].encode()).decode()})
if response.status_code != 201:
return self._submit_error('Could not create file.'+response.text)
else:
response = requests.get(endpoint_url('/repos/%s/contents/%s' % (fork_name, data['file_path'])),
params={'ref': data['commit_id']})
if response.status_code != 200:
return self._submit_error('Could not get file.')
file_sha = response.json()['sha']
if data['action'] == 'edit':
response = requests.put(endpoint_url('/repos/%s/contents/%s' % (fork_name, data['file_path'])),
json={'branch': branch_name, 'message': data['commit_msg'], 'sha': file_sha,
'content': base64.b64encode(data['content'].encode()).decode()})
if response.status_code != 200:
return self._submit_error('Could not update file.')
elif data['action'] == 'delete':
response = requests.put(endpoint_url('/repos/%s/contents/%s' % (fork_name, data['file_path'])),
json={'branch': branch_name, 'message': data['commit_msg'], 'sha': file_sha})
if response.status_code != 200:
return self._submit_error('Could not delete file.' + response.text)
# Create pull request
response = requests.post(endpoint_url('/repos/%s/pulls' % repo_name),
json={'base': 'master', 'head': '%s:%s' % (user['login'], branch_name),
'title': data['commit_msg']})
if response.status_code != 201:
return self._submit_error('Could not delete file.' + response.text)
merge_request = response.json()
return {
'success': True,
'url': merge_request['html_url']
}
def get_user_id_with_access_token(self, access_token):
# Todo: Implement this
return None

150
src/c3nav/editor/hosters/gitlab.py
View file

@ -1,150 +0,0 @@
import string
import uuid
from urllib.parse import urlencode, urljoin
import requests
from django.core.exceptions import SuspiciousOperation
from django.utils.crypto import get_random_string
from c3nav.editor.hosters.base import Hoster
from c3nav.mapdata.models.package import Package
class GitlabHoster(Hoster):
title = 'Gitlab'
def __init__(self, app_id, app_secret, **kwargs):
super().__init__(**kwargs)
self._app_id = app_id
self._app_secret = app_secret
def get_endpoint(self, path):
return urljoin(self.base_url, path)
def get_auth_uri(self, request):
oauth_csrf_token = get_random_string(42, string.ascii_letters+string.digits)
self._get_session_data(request)['oauth_csrf_token'] = oauth_csrf_token
callback_uri = self._get_callback_uri(request)
self._get_session_data(request)['callback_uri'] = callback_uri
return self.get_endpoint('/oauth/authorize?%s' % urlencode((
('client_id', self._app_id),
('redirect_uri', callback_uri),
('response_type', 'code'),
('state', oauth_csrf_token),
)))
def handle_callback_request(self, request):
code = request.GET.get('code')
state = request.GET.get('state')
if code is None or state is None:
raise SuspiciousOperation('Missing parameters.')
session_data = self._get_session_data(request)
if session_data.get('oauth_csrf_token') != state:
raise SuspiciousOperation('OAuth CSRF token mismatch')
session_data.pop('oauth_csrf_token')
callback_uri = session_data.pop('callback_uri')
self.request_access_token(request, code, state, callback_uri)
def do_request_access_token(self, code, state, callback_uri):
response = requests.post(self.get_endpoint('/oauth/token'), data={
'client_id': self._app_id,
'client_secret': self._app_secret,
'code': code,
'grant_type': 'authorization_code',
'redirect_uri': callback_uri,
'state': state,
}).json()
if 'error' in response:
return {
'state': 'logged_out',
'error': '%s: %s' % (response['error'], response['error_description'])
}
return {
'state': 'logged_in',
'access_token': response['access_token']
}
def do_check_access_token(self, access_token):
response = requests.get(self.get_endpoint('/user'), headers={'Authorization': 'Bearer '+access_token})
if response.status_code != 200:
return {'state': 'logged_out'}
return {'state': 'logged_in'}
def do_submit_edit(self, access_token, data):
# Get endpoint URL with access token
def endpoint_url(endpoint):
return self.base_url + 'api/v3' + endpoint + '?access_token=' + access_token
# Get Package from db
try:
package = Package.objects.get(name=data['package_name'])
except Package.DoesNotExist:
return self._submit_error('Could not find package.')
# Get project name on this host, e.g. c3nav/c3nav
project_name = '/'.join(package.home_repo[len(self.base_url):].split('/')[:2])
# Get project from Gitlab API
response = requests.get(endpoint_url('/projects/' + project_name.replace('/', '%2F')))
if response.status_code != 200:
return self._submit_error('Could not find project.')
project = response.json()
# Create branch
branch_name = 'editor-%s' % uuid.uuid4()
response = requests.post(endpoint_url('/projects/%d/repository/branches' % project['id']),
data={'branch_name': branch_name, 'ref': data['commit_id']})
if response.status_code != 201:
return self._submit_error('Could not create branch.')
# Make commit
if data['action'] == 'create':
response = requests.post(endpoint_url('/projects/%d/repository/files' % project['id']),
data={'branch_name': branch_name, 'encoding': 'text', 'content': data['content'],
'file_path': data['file_path'], 'commit_message': data['commit_msg']})
if response.status_code != 201:
return self._submit_error('Could not create file.')
elif data['action'] == 'edit':
response = requests.put(endpoint_url('/projects/%d/repository/files' % project['id']),
data={'branch_name': branch_name, 'encoding': 'text', 'content': data['content'],
'file_path': data['file_path'], 'commit_message': data['commit_msg']})
if response.status_code != 200:
return self._submit_error('Could not update file.')
elif data['action'] == 'delete':
response = requests.delete(endpoint_url('/projects/%d/repository/files' % project['id']),
data={'branch_name': branch_name, 'file_path': data['file_path'],
'commit_message': data['commit_msg']})
if response.status_code != 200:
return self._submit_error('Could not delete file.' + response.text)
# Create merge request
response = requests.post(endpoint_url('/projects/%d/merge_requests' % project['id']),
data={'source_branch': branch_name, 'target_branch': 'master',
'title': data['commit_msg']})
if response.status_code != 201:
return self._submit_error('Could not create merge request.')
merge_request = response.json()
return {
'success': True,
'url': merge_request['web_url']
}
def get_user_id_with_access_token(self, access_token):
if not access_token.strip():
return None
response = requests.get(self.base_url + 'api/v3/user?private_token=' + access_token)
if response.status_code != 200:
return None
return self.base_url+'user/'+str(response.json()['id'])

33
src/c3nav/editor/serializers.py
View file

@ -1,33 +0,0 @@
from rest_framework import serializers
class HosterSerializer(serializers.Serializer):
name = serializers.CharField()
base_url = serializers.CharField()
class TaskSerializer(serializers.Serializer):
id = serializers.CharField()
started = serializers.SerializerMethodField()
done = serializers.SerializerMethodField()
success = serializers.SerializerMethodField()
result = serializers.SerializerMethodField()
error = serializers.SerializerMethodField()
def get_started(self, obj):
return obj.status != 'PENDING'
def get_done(self, obj):
return obj.ready()
def get_success(self, obj):
return (obj.successful() and obj.result['success']) if obj.ready() else None
def get_result(self, obj):
return obj.result if obj.ready() and obj.successful() else None
def get_error(self, obj):
success = self.get_success(obj)
if success is not False:
return None
return 'Internal Error' if not obj.successful() else obj.result['error']

3
src/c3nav/editor/static/editor/css/finalize.css
View file

@ -1,3 +0,0 @@
.hoster-state, #hoster #error {
display:none;
}

75
src/c3nav/editor/static/editor/js/editor.js
View file

@ -31,68 +31,12 @@ editor = {
editor.init_geometries();
editor.init_sidebar();
editor.get_packages();
editor.get_sources();
editor.get_levels();
},
// packages
packages: {},
_shown_packages: [],
_packages_control: null,
get_packages: function() {
editor._packages_control = L.control.layers().addTo(editor.map);
$(editor._packages_control._layersLink).text('Packages');
// load packages
$.getJSON('/api/packages/', function (packages) {
var bounds = [[0, 0], [0, 0]];
var pkg, layer;
for (var i = 0; i < packages.length; i++) {
pkg = packages[i];
editor.packages[pkg.name] = pkg;
layer = L.circle([-200, -200], 0.1);
layer._c3nav_package = pkg.name;
layer.on('add', editor._add_package_layer);
layer.on('remove', editor._remove_package_layer);
layer.addTo(editor.map);
editor._packages_control.addOverlay(layer, pkg.name);
editor._shown_packages.push(pkg.name);
if (pkg.bounds === null) continue;
bounds = [[Math.min(bounds[0][0], pkg.bounds[0][0]), Math.min(bounds[0][1], pkg.bounds[0][1])],
[Math.max(bounds[1][0], pkg.bounds[1][0]), Math.max(bounds[1][1], pkg.bounds[1][1])]];
}
editor.map.setMaxBounds(bounds);
editor.map.fitBounds(bounds, {padding: [30, 50]});
});
},
_add_package_layer: function(e) {
var pkg = e.target._c3nav_package;
var i = editor._shown_packages.indexOf(pkg);
if (i == -1) {
if (editor._loading_geometry) {
e.target.remove();
return;
}
editor._loading_geometry = true;
editor._shown_packages.push(pkg);
editor.get_geometries();
}
},
_remove_package_layer: function(e) {
var pkg = e.target._c3nav_package;
var i = editor._shown_packages.indexOf(pkg);
if (i > -1) {
if (editor._loading_geometry) {
e.target.addTo(map);
return;
}
editor._loading_geometry = true;
editor._shown_packages.splice(i, 1);
editor.get_geometries();
}
bounds = [[0.0, 0.0], [240.0, 400.0]];
editor.map.setMaxBounds(bounds);
editor.map.fitBounds(bounds, {padding: [30, 50]});
},
// sources
@ -256,11 +200,7 @@ editor = {
geometrytypes += '&type=' + editor._geometry_types[i];
}
}
packages = '';
for (var i = 0; i < editor._shown_packages.length; i++) {
packages += '&package=' + editor._shown_packages[i];
}
$.getJSON('/api/geometries/?level='+String(editor._level)+geometrytypes+packages, function(geometries) {
$.getJSON('/api/geometries/?level='+String(editor._level)+geometrytypes, function(geometries) {
editor._geometries_layer = L.geoJSON(geometries, {
style: editor._get_geometry_style,
onEachFeature: editor._register_geojson_feature
@ -396,13 +336,6 @@ editor = {
id_name.select();
}
var package_field = mapeditcontrols.find('select[name=package]');
if (package_field.length) {
if (package_field.val() === '' && editor._shown_packages.length == 1) {
package_field.val(editor._shown_packages[0]);
}
}
var geometry_field = mapeditcontrols.find('input[name=geometry]');
if (geometry_field.length) {
var form = geometry_field.closest('form');

84
src/c3nav/editor/static/editor/js/finalize.js
View file

@ -1,84 +0,0 @@
finalize = {
hoster: null,
state: 'checking',
submittask: null,
init: function() {
finalize.hoster = $('#hoster').attr('data-name');
finalize._set_state('checking');
finalize._check_hoster();
sessionStorage.setItem('finalize-data', finalize.get_data());
$('button[data-oauth]').click(finalize._click_oauth_btn);
$('button[data-commit]').click(finalize._click_commit_btn);
},
get_data: function() {
return $('#data').val();
},
_check_hoster: function() {
$.getJSON('/api/hosters/'+finalize.hoster+'/state/', function(data) {
if (data.state == 'checking') {
window.setTimeout(finalize._check_hoster, 700);
} else {
$('#error').text(data.error).toggle(data.error !== null);
finalize._set_state(data.state);
}
});
},
_set_state: function(state) {
finalize.state = state;
$('.hoster-state').hide().filter('[data-state='+state+']').show();
$('#alternatively').toggle(['progress', 'done'].indexOf(state) == -1);
},
_click_oauth_btn: function() {
finalize._set_state('oauth');
$.ajax({
type: "POST",
url: '/api/hosters/'+finalize.hoster+'/auth_uri/',
dataType: 'json',
headers: {'X-CSRFToken': $('[name=csrfmiddlewaretoken]').val()},
success: function(data) {
window.location = data.auth_uri;
}
});
},
_click_commit_btn: function() {
var commit_msg = $.trim($('#commit_msg').val());
if (commit_msg == '') return;
$('#error').hide();
finalize._set_state('progress');
$.ajax({
type: "POST",
url: '/api/hosters/'+finalize.hoster+'/submit/',
data: {
'data': finalize.get_data(),
'commit_msg': commit_msg
},
dataType: 'json',
headers: {'X-CSRFToken': $('[name=csrfmiddlewaretoken]').val()},
success: finalize.handle_task_data
});
},
handle_task_data: function(data) {
finalize.submittask = data.id;
if (data.done) {
if (!data.success) {
$('#error').text(data.error).show();
finalize._set_state('logged_in');
} else {
$('#pull_request_link').attr('href', data.result.url).text(data.result.url);
finalize._set_state('done');
}
} else {
window.setTimeout(finalize._check_submittask, 700);
}
},
_check_submittask: function() {
$.getJSON('/api/submittasks/'+finalize.submittask+'/', finalize.handle_task_data);
}
};
if ($('#hoster').length) {
finalize.init();
}
if ($('#finalize-redirect').length) {
$('form').append($('<input type="hidden" name="data">').val(sessionStorage.getItem('finalize-data'))).submit();
}

19
src/c3nav/editor/tasks.py
View file

@ -1,19 +0,0 @@
from c3nav.celery import app
@app.task()
def request_access_token_task(hoster, *args, **kwargs):
from c3nav.editor.hosters import hosters
return hosters[hoster].do_request_access_token(*args, **kwargs)
@app.task()
def check_access_token_task(hoster, access_token):
from c3nav.editor.hosters import hosters
return hosters[hoster].do_check_access_token(access_token)
@app.task()
def submit_edit_task(hoster, access_token, data):
from c3nav.editor.hosters import hosters
return hosters[hoster].do_submit_edit(access_token, data)

2
src/c3nav/editor/templates/editor/base.html
View file

@ -11,7 +11,6 @@
<link href="{% static 'bootstrap/css/bootstrap.css' %}" rel="stylesheet">
<link href="{% static 'leaflet/leaflet.css' %}" rel="stylesheet">
<link href="{% static 'editor/css/editor.css' %}" rel="stylesheet">
<link href="{% static 'editor/css/finalize.css' %}" rel="stylesheet">
{% endcompress %}
</head>
@ -38,7 +37,6 @@
<script type="text/javascript" src="{% static 'leaflet/leaflet.js' %}"></script>
<script type="text/javascript" src="{% static 'leaflet/leaflet.editable.js' %}"></script>
<script type="text/javascript" src="{% static 'editor/js/editor.js' %}"></script>
<script type="text/javascript" src="{% static 'editor/js/finalize.js' %}"></script>
{% endcompress %}
</body>
</html>

93
src/c3nav/editor/templates/editor/finalize.html
View file

@ -1,93 +0,0 @@
{% extends 'editor/base.html' %}
{% load static %}
{% load bootstrap3 %}
{% block content %}
<input type="hidden" id="data" value="{{ data }}">
{% csrf_token %}
{% if hoster %}
<div id="hoster" data-name="{{ hoster.name }}">
<div class="alert alert-danger" role="alert" id="error"></div>
<noscript>
<h2>Please enable Javascript to propose your edit.</h2>
</noscript>
<div class="hoster-state" data-state="checking">
<h2>Sign in with {{ hoster.title }}</h2>
<p><img src="{% static 'img/loader.gif' %}"></p>
<p><em>Checking authentication, please wait…</em></p>
</div>
<div class="hoster-state" data-state="logged_out">
<h2>Sign in with {{ hoster.title }}</h2>
<p>Please sign in to continue and propose your edit.</p>
<p>
<button class="btn btn-lg btn-primary" data-oauth>Sign in with {{ hoster.title }}</button><br>
<small><em>{{ hoster.name }} – {{ hoster.base_url }}</em></small>
</p>
</div>
<div class="hoster-state" data-state="missing_permissions">
<h2>Missing {{ hoster.title }} Permissions</h2>
<p>c3nav is missing permissions that it needs to propose your edit.</p>
<p>Please click the button below to grant the missing permissions.</p>
<p>
<button class="btn btn-lg btn-primary" data-oauth>Sign in with {{ hoster.title }}</button><br>
<small><em>{{ hoster.name }} – {{ hoster.base_url }}</em></small>
</p>
</div>
<div class="hoster-state" data-state="oauth">
<h2>Redirecting…</h2>
<p><img src="{% static 'img/loader.gif' %}"></p>
<p><em>You will be redirected to {{ hoster.title }}…</em></p>
</div>
<div class="hoster-state" data-state="logged_in">
<h2>Propose Changes</h2>
<p>Please provide a short helpful title for your change.</p>
<p>
<input class="form-control" id="commit_msg" maxlength="100" type="text" value="{{ commit_msg }}">
</p>
<p>
<button class="btn btn-lg btn-primary" data-commit>Create Pull Request</button><br>
<small><em>
{{ hoster.name }} – {{ hoster.base_url }}
</em></small>
</p>
</div>
<div class="hoster-state" data-state="progress">
<h2>Proposing Changes…</h2>
<p><img src="{% static 'img/loader.gif' %}"></p>
<p><em>Proposing your changes, please wait…</em></p>
</div>
<div class="hoster-state" data-state="done">
<h2>Pull Request created</h2>
<p>Click the link below to go to the pull request:</p>
<p><a href="" id="pull_request_link"></a></p>
<p><em>You really should do so if you want to add a description.</em></p>
</div>
<p id="alternatively">Alternatively, you can copy your edit below and send it to the maps maintainer.</p>
</div>
{% else %}
<h2>Copy your edit</h2>
<p>In order to propose your edit, please copy it and send it to the maps maintainer.</p>
<p><em>You are seeing this message because there is no hoster defined for this map package.</em></p>
{% endif %}
<h3>Your Edit</h3>
<p><strong>Map Package:</strong> {{ package_name }}</p>
<p>
<strong>
{% if action == 'create' %}
Create file:
{% elif action == 'edit' %}
Edit file:
{% elif action == 'delete' %}
Delete file:
{% endif %}
</strong>
<code>{{ file_path }}</code>
</p>
<p><strong>Parent commit id:</strong> <code>{{ commit_id }}</code></p>
{% if action != 'delete' %}
<p>
<strong>New file contents:</strong>
<pre>{{ file_contents }}</pre>
</p>
{% endif %}
{% endblock %}

9
src/c3nav/editor/templates/editor/finalize_redirect.html
View file

@ -1,9 +0,0 @@
{% extends 'editor/base.html' %}
{% load static %}
{% block content %}
<form action="{% url 'editor.finalize' %}" method="POST" id="finalize-redirect">
{% csrf_token %}
<img src="{% static 'img/loader.gif' %}">
Redirecting…
</form>
{% endblock %}

11
src/c3nav/editor/templates/editor/mapitem_success.html
View file

@ -1,11 +1,2 @@
{% load static %}
{% if data %}
<form action="{% url 'editor.finalize' %}" method="POST" name="redirect">
{% csrf_token %}
<input type="hidden" name="data" value="{{ data }}">
<img src="{% static 'img/loader.gif' %}">
Redirecting…
</form>
{% else %}
<span data-redirect="{% url 'editor.mapitems.level' mapitem_type=mapitem_type level='LEVEL' %}"></span>
{% endif %}
<span data-redirect="{% url 'editor.mapitems.level' mapitem_type=mapitem_type level='LEVEL' %}"></span>

9
src/c3nav/editor/templates/editor/oauth_callback.html
View file

@ -1,9 +0,0 @@
{% extends 'editor/base.html' %}
{% load static %}
{% block content %}
<form action="{% url 'editor.finalize' %}" method="POST" name="redirect">
{% csrf_token %}
<input type="hidden" name="data" value="{{ data }}">
Redirecting…
</form>
{% endblock %}

4
src/c3nav/editor/urls.py
View file

@ -1,7 +1,7 @@
from django.conf.urls import url
from django.views.generic import TemplateView
from c3nav.editor.views import edit_mapitem, finalize, list_mapitems, list_mapitemtypes, oauth_callback
from c3nav.editor.views import edit_mapitem, list_mapitems, list_mapitemtypes
urlpatterns = [
url(r'^$', TemplateView.as_view(template_name='editor/map.html'), name='editor.index'),
@ -10,6 +10,4 @@ urlpatterns = [
url(r'^mapitems/(?P<mapitem_type>[^/]+)/list/(?P<level>[^/]+)/$', list_mapitems, name='editor.mapitems.level'),
url(r'^mapitems/(?P<mapitem_type>[^/]+)/add/$', edit_mapitem, name='editor.mapitems.add'),
url(r'^mapitems/(?P<mapitem_type>[^/]+)/edit/(?P<name>[^/]+)/$', edit_mapitem, name='editor.mapitems.edit'),
url(r'^finalize/$', finalize, name='editor.finalize'),
url(r'^oauth/(?P<hoster>[^/]+)/callback$', oauth_callback, name='editor.oauth.callback')
]

87
src/c3nav/editor/views.py
View file

@ -1,16 +1,11 @@
from django.conf import settings
from django.core import signing
from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.core.signing import BadSignature
from django.core.exceptions import PermissionDenied
from django.http.response import Http404
from django.shortcuts import get_object_or_404, redirect, render
from django.utils import translation
from c3nav.access.apply import can_access_package, filter_queryset_by_access
from c3nav.editor.hosters import get_hoster_for_package, hosters
from c3nav.access.apply import filter_queryset_by_access, can_access
from c3nav.mapdata.models import AreaLocation
from c3nav.mapdata.models.base import MAPITEM_TYPES
from c3nav.mapdata.models.package import Package
def list_mapitemtypes(request, level):
@ -81,7 +76,7 @@ def edit_mapitem(request, mapitem_type, name=None):
if name is not None:
# Edit existing map item
mapitem = get_object_or_404(mapitemtype, name=name)
if not can_access_package(request, mapitem.package):
if not can_access(request, mapitem):
raise PermissionDenied
new = mapitem is None
@ -92,18 +87,8 @@ def edit_mapitem(request, mapitem_type, name=None):
# Delete this mapitem!
if request.POST.get('delete_confirm') == '1':
if not settings.DIRECT_EDITING:
with translation.override('en'):
commit_msg = 'Deleted %s: %s' % (mapitemtype._meta.verbose_name, mapitem.title)
return render(request, 'editor/mapitem_success.html', {
'data': signing.dumps({
'type': 'editor.edit',
'action': 'delete',
'package_name': mapitem.package.name,
'commit_id': mapitem.package.commit_id,
'commit_msg': commit_msg,
'file_path': mapitem.get_filename(),
})
})
# todo: suggest changes
raise NotImplementedError
mapitem.delete()
return render(request, 'editor/mapitem_success.html', {
@ -119,8 +104,6 @@ def edit_mapitem(request, mapitem_type, name=None):
form = mapitemtype.EditorForm(instance=mapitem, data=request.POST, request=request)
if form.is_valid():
# Update/create mapitem
commit_type = 'Created' if mapitem is None else 'Updated'
action = 'create' if mapitem is None else 'edit'
mapitem = form.save(commit=False)
if form.titles is not None:
@ -131,20 +114,7 @@ def edit_mapitem(request, mapitem_type, name=None):
if not settings.DIRECT_EDITING:
# todo: suggest changes
content = ''
with translation.override('en'):
commit_msg = '%s %s: %s' % (commit_type, mapitemtype._meta.verbose_name, mapitem.title)
return render(request, 'editor/mapitem_success.html', {
'data': signing.dumps({
'type': 'editor.edit',
'action': action,
'package_name': mapitem.package.name,
'commit_id': mapitem.package.commit_id,
'commit_msg': commit_msg,
'file_path': mapitem.get_filename(),
'content': content,
})
})
raise NotImplementedError
mapitem.save()
form.save_m2m()
@ -165,48 +135,3 @@ def edit_mapitem(request, mapitem_type, name=None):
'path': request.path,
'new': new
})
def finalize(request):
if request.method != 'POST':
return render(request, 'editor/finalize_redirect.html', {})
if 'data' not in request.POST:
raise SuspiciousOperation('Missing data.')
raw_data = request.POST['data']
try:
data = signing.loads(raw_data)
except BadSignature:
raise SuspiciousOperation('Bad Signature.')
if data['type'] != 'editor.edit':
raise SuspiciousOperation('Wrong data type.')
package = Package.objects.filter(name=data['package_name']).first()
hoster = None
if package is not None:
hoster = get_hoster_for_package(package)
hoster.check_state(request)
return render(request, 'editor/finalize.html', {
'hoster': hoster,
'data': raw_data,
'action': data['action'],
'commit_id': data['commit_id'],
'commit_msg': data['commit_msg'],
'package_name': data['package_name'],
'file_path': data['file_path'],
'file_contents': data.get('content')
})
def oauth_callback(request, hoster):
hoster = hosters.get(hoster)
if hoster is None:
raise Http404
hoster.handle_callback_request(request)
return render(request, 'editor/finalize_redirect.html', {})

39
src/c3nav/mapdata/api.py
View file

@ -8,14 +8,14 @@ from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
from c3nav.access.apply import filter_arealocations_by_access, filter_queryset_by_access, get_unlocked_packages_names
from c3nav.access.apply import filter_arealocations_by_access, filter_queryset_by_access
from c3nav.mapdata.lastupdate import get_last_mapdata_update
from c3nav.mapdata.models import GEOMETRY_MAPITEM_TYPES, AreaLocation, Level, LocationGroup, Package, Source
from c3nav.mapdata.models import GEOMETRY_MAPITEM_TYPES, AreaLocation, Level, LocationGroup, Source
from c3nav.mapdata.models.geometry import DirectedLineGeometryMapItemWithLevel
from c3nav.mapdata.search import get_location
from c3nav.mapdata.serializers.main import LevelSerializer, PackageSerializer, SourceSerializer
from c3nav.mapdata.serializers.main import LevelSerializer, SourceSerializer
from c3nav.mapdata.utils.cache import (CachedReadOnlyViewSetMixin, cache_mapdata_api_response, get_bssid_areas_cached,
get_levels_cached, get_packages_cached)
get_levels_cached)
class GeometryTypeViewSet(ViewSet):
@ -36,7 +36,7 @@ class GeometryTypeViewSet(ViewSet):
class GeometryViewSet(ViewSet):
"""
List all geometries.
You can filter by adding a level GET parameter or one or more package or type GET parameters.
You can filter by adding a level GET parameter.
"""
def list(self, request):
types = set(request.GET.getlist('type'))
@ -53,33 +53,23 @@ class GeometryViewSet(ViewSet):
if level_name in levels_cached:
level = levels_cached[level_name]
packages_cached = get_packages_cached()
package_names = set(request.GET.getlist('package')) & set(get_unlocked_packages_names(request))
packages = [packages_cached[name] for name in package_names if name in packages_cached]
if len(packages) == len(packages_cached):
packages = []
package_ids = sorted([package.id for package in packages])
cache_key = '__'.join((
','.join([str(i) for i in types]),
str(level.id) if level is not None else '',
','.join([str(i) for i in package_ids]),
))
return self._list(request, types=types, level=level, packages=packages, add_cache_key=cache_key)
return self._list(request, types=types, level=level, add_cache_key=cache_key)
@staticmethod
def compare_by_location_type(x: AreaLocation, y: AreaLocation):
return AreaLocation.LOCATION_TYPES.index(x.location_type) - AreaLocation.LOCATION_TYPES.index(y.location_type)
@cache_mapdata_api_response()
def _list(self, request, types, level, packages):
def _list(self, request, types, level):
results = []
for t in types:
mapitemtype = GEOMETRY_MAPITEM_TYPES[t]
queryset = mapitemtype.objects.all()
if packages:
queryset = queryset.filter(package__in=packages)
if level:
if hasattr(mapitemtype, 'level'):
queryset = queryset.filter(level=level)
@ -90,7 +80,7 @@ class GeometryViewSet(ViewSet):
queryset = filter_queryset_by_access(request, queryset)
queryset = queryset.order_by('name')
for field_name in ('package', 'level', 'crop_to_level', 'elevator'):
for field_name in ('level', 'crop_to_level', 'elevator'):
if hasattr(mapitemtype, field_name):
queryset = queryset.select_related(field_name)
@ -109,17 +99,6 @@ class GeometryViewSet(ViewSet):
return Response(results)
class PackageViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
Retrieve packages the map consists of.
"""
queryset = Package.objects.all()
serializer_class = PackageSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
ordering = ('name',)
class LevelViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
List and retrieve levels.
@ -140,7 +119,6 @@ class SourceViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
lookup_field = 'name'
lookup_value_regex = '[^/]+'
ordering = ('name',)
include_package_access = True
def get_queryset(self):
return filter_queryset_by_access(self.request, super().get_queryset().all())
@ -163,7 +141,6 @@ class LocationViewSet(ViewSet):
"""
# We don't cache this, because it depends on access_list
lookup_field = 'name'
include_package_access = True
@staticmethod
def _filter(queryset):

1
src/c3nav/mapdata/models/__init__.py
View file

@ -1,5 +1,4 @@
from .level import Level # noqa
from .package import Package # noqa
from .source import Source # noqa
from .collections import Elevator # noqa
from .geometry import GeometryMapItemWithLevel, GEOMETRY_MAPITEM_TYPES # noqa

1
src/c3nav/mapdata/models/base.py
View file

@ -20,7 +20,6 @@ class MapItemMeta(ModelBase):
class MapItem(models.Model, metaclass=MapItemMeta):
name = models.SlugField(_('Name'), unique=True, max_length=50)
package = models.ForeignKey('mapdata.Package', on_delete=models.CASCADE, verbose_name=_('map package'))
EditorForm = None

12
src/c3nav/mapdata/models/geometry.py
View file

@ -37,7 +37,6 @@ class GeometryMapItem(MapItem, metaclass=GeometryMapItemMeta):
return OrderedDict((
('type', self.__class__.__name__.lower()),
('name', self.name),
('package', self.package.name),
))
def to_geojson(self):
@ -122,6 +121,11 @@ class Room(GeometryMapItemWithLevel):
verbose_name_plural = _('Rooms')
default_related_name = 'rooms'
def get_geojson_properties(self):
result = super().get_geojson_properties()
result['public'] = self.public
return result
class Outside(GeometryMapItemWithLevel):
"""
@ -135,6 +139,11 @@ class Outside(GeometryMapItemWithLevel):
verbose_name_plural = _('Outside Areas')
default_related_name = 'outsides'
def get_geojson_properties(self):
result = super().get_geojson_properties()
result['public'] = self.public
return result
class StuffedArea(GeometryMapItemWithLevel):
"""
@ -290,6 +299,7 @@ class ElevatorLevel(GeometryMapItemWithLevel):
def get_geojson_properties(self):
result = super().get_geojson_properties()
result['public'] = self.public
result['elevator'] = self.elevator.name
result['button'] = self.button
return result

5
src/c3nav/mapdata/models/level.py
View file

@ -55,14 +55,11 @@ class LevelGeometries():
self.level = level
self.only_public = only_public
from c3nav.access.apply import get_public_packages
self.public_packages = get_public_packages()
def query(self, name):
queryset = getattr(self.level, name)
if not self.only_public:
return queryset.all()
return queryset.filter(package__in=self.public_packages)
return queryset.filter(public=True)
@cached_property
def raw_rooms(self):

2
src/c3nav/mapdata/models/locations.py
View file

@ -109,7 +109,7 @@ class AreaLocation(LocationModelMixin, GeometryMapItemWithLevel):
)
LOCATION_TYPES_ORDER = tuple(name for name, title in LOCATION_TYPES)
ROUTING_INCLUSIONS = (
('default', _('Default, include if map package is unlocked')),
('default', _('Default, include it is unlocked')),
('allow_avoid', _('Included, but allow excluding')),
('allow_include', _('Avoided, but allow including')),
('needs_permission', _('Excluded, needs permission to include')),

49
src/c3nav/mapdata/models/package.py
View file

@ -1,49 +0,0 @@
from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.lastupdate import set_last_mapdata_update
class Package(models.Model):
"""
A c3nav map package
"""
name = models.SlugField(_('package identifier'), unique=True, max_length=50,
help_text=_('e.g. de.c3nav.33c3.base'))
depends = models.ManyToManyField('Package')
home_repo = models.URLField(_('URL to the home git repository'), null=True)
commit_id = models.CharField(_('current commit id'), max_length=40, null=True)
bottom = models.DecimalField(_('bottom coordinate'), null=True, max_digits=6, decimal_places=2)
left = models.DecimalField(_('left coordinate'), null=True, max_digits=6, decimal_places=2)
top = models.DecimalField(_('top coordinate'), null=True, max_digits=6, decimal_places=2)
right = models.DecimalField(_('right coordinate'), null=True, max_digits=6, decimal_places=2)
directory = models.CharField(_('folder name'), max_length=100)
class Meta:
verbose_name = _('Map Package')
verbose_name_plural = _('Map Packages')
default_related_name = 'packages'
@property
def package(self):
return self
@property
def bounds(self):
if self.bottom is None:
return None
return (float(self.bottom), float(self.left)), (float(self.top), float(self.right))
@property
def public(self):
return self.name in settings.PUBLIC_PACKAGES
def save(self, *args, **kwargs):
with set_last_mapdata_update():
super().save(*args, **kwargs)
def __str__(self):
return self.name

27
src/c3nav/mapdata/serializers/main.py
View file

@ -1,36 +1,15 @@
from rest_framework import serializers
from c3nav.editor.hosters import get_hoster_for_package
from c3nav.mapdata.models import Level, Package, Source
class PackageSerializer(serializers.ModelSerializer):
hoster = serializers.SerializerMethodField()
depends = serializers.SlugRelatedField(slug_field='name', many=True, read_only=True)
class Meta:
model = Package
fields = ('name', 'home_repo', 'commit_id', 'depends', 'bounds', 'public', 'hoster')
def get_depends(self, obj):
return self.recursive_value(PackageSerializer, obj.depends, many=True)
def get_hoster(self, obj):
hoster = get_hoster_for_package(obj)
return hoster.name if hoster else None
from c3nav.mapdata.models import Level, Source
class LevelSerializer(serializers.ModelSerializer):
package = serializers.SlugRelatedField(slug_field='name', read_only=True)
class Meta:
model = Level
fields = ('name', 'altitude', 'package')
fields = ('name', 'altitude')
class SourceSerializer(serializers.ModelSerializer):
package = serializers.SlugRelatedField(slug_field='name', read_only=True)
class Meta:
model = Source
fields = ('name', 'package', 'bounds')
fields = ('name', 'bounds')

14
src/c3nav/mapdata/utils/cache.py
View file

@ -60,16 +60,8 @@ def cache_mapdata_api_response(timeout=900):
class CachedReadOnlyViewSetMixin():
include_package_access = False
def _get_unlocked_packages_ids(self, request):
from c3nav.access.apply import get_unlocked_packages
return ','.join(str(i) for i in sorted(package.id for package in get_unlocked_packages(request)))
def _get_add_cache_key(self, request, add_cache_key=''):
cache_key = add_cache_key
if self.include_package_access:
cache_key += '__'+self._get_unlocked_packages_ids(request)
return cache_key
def list(self, request, *args, **kwargs):
@ -95,12 +87,6 @@ def get_levels_cached():
return OrderedDict((level.name, level) for level in Level.objects.all())
@cache_result('c3nav__mapdata__packages')
def get_packages_cached():
from c3nav.mapdata.models import Package
return {package.name: package for package in Package.objects.all()}
@cache_result('c3nav__mapdata__bssids')
def get_bssid_areas_cached():
from c3nav.mapdata.models import AreaLocation

9
src/c3nav/mapdata/utils/misc.py
View file

@ -1,21 +1,16 @@
import os
from django.conf import settings
from django.db.models import Max, Min
from shapely.geometry import box
from shapely.ops import cascaded_union
from c3nav.mapdata.models import Package
from c3nav.mapdata.utils.cache import cache_result
@cache_result('c3nav__mapdata__dimensions')
def get_dimensions():
aggregate = Package.objects.all().aggregate(Max('right'), Min('left'), Max('top'), Min('bottom'))
return (
float(aggregate['right__max'] - aggregate['left__min']),
float(aggregate['top__max'] - aggregate['bottom__min']),
)
# todo calculate this
return (400, 240)
@cache_result('c3nav__mapdata__render_dimensions')

220
src/c3nav/routing/api.py
View file

@ -1,220 +0,0 @@
import hashlib
import json
import mimetypes
import os
from collections import OrderedDict
from django.conf import settings
from django.core.files import File
from django.http import Http404, HttpResponse, HttpResponseNotModified
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
from c3nav.access.apply import filter_arealocations_by_access, filter_queryset_by_access, get_unlocked_packages_names
from c3nav.mapdata.lastupdate import get_last_mapdata_update
from c3nav.mapdata.models import GEOMETRY_MAPITEM_TYPES, AreaLocation, Level, LocationGroup, Package, Source
from c3nav.mapdata.models.geometry import DirectedLineGeometryMapItemWithLevel
from c3nav.mapdata.search import get_location
from c3nav.mapdata.serializers.main import LevelSerializer, PackageSerializer, SourceSerializer
from c3nav.mapdata.utils.cache import (CachedReadOnlyViewSetMixin, cache_mapdata_api_response, get_bssid_areas_cached,
get_levels_cached, get_packages_cached)
class RoutingmetryTypeViewSet(ViewSet):
"""
Lists all geometry types.
"""
@cache_mapdata_api_response()
def list(self, request):
return Response([
OrderedDict((
('name', name),
('title', str(mapitemtype._meta.verbose_name)),
('title_plural', str(mapitemtype._meta.verbose_name_plural)),
)) for name, mapitemtype in GEOMETRY_MAPITEM_TYPES.items()
])
class GeometryViewSet(ViewSet):
"""
List all geometries.
You can filter by adding a level GET parameter or one or more package or type GET parameters.
"""
def list(self, request):
types = set(request.GET.getlist('type'))
valid_types = list(GEOMETRY_MAPITEM_TYPES.keys())
if not types:
types = valid_types
else:
types = [t for t in valid_types if t in types]
level = None
if 'level' in request.GET:
levels_cached = get_levels_cached()
level_name = request.GET['level']
if level_name in levels_cached:
level = levels_cached[level_name]
packages_cached = get_packages_cached()
package_names = set(request.GET.getlist('package')) & set(get_unlocked_packages_names(request))
packages = [packages_cached[name] for name in package_names if name in packages_cached]
if len(packages) == len(packages_cached):
packages = []
package_ids = sorted([package.id for package in packages])
cache_key = '__'.join((
','.join([str(i) for i in types]),
str(level.id) if level is not None else '',
','.join([str(i) for i in package_ids]),
))
return self._list(request, types=types, level=level, packages=packages, add_cache_key=cache_key)
@staticmethod
def compare_by_location_type(x: AreaLocation, y: AreaLocation):
return AreaLocation.LOCATION_TYPES.index(x.location_type) - AreaLocation.LOCATION_TYPES.index(y.location_type)
@cache_mapdata_api_response()
def _list(self, request, types, level, packages):
results = []
for t in types:
mapitemtype = GEOMETRY_MAPITEM_TYPES[t]
queryset = mapitemtype.objects.all()
if packages:
queryset = queryset.filter(package__in=packages)
if level:
if hasattr(mapitemtype, 'level'):
queryset = queryset.filter(level=level)
elif hasattr(mapitemtype, 'levels'):
queryset = queryset.filter(levels=level)
else:
queryset = queryset.none()
queryset = filter_queryset_by_access(request, queryset)
queryset = queryset.order_by('name')
for field_name in ('package', 'level', 'crop_to_level', 'elevator'):
if hasattr(mapitemtype, field_name):
queryset = queryset.select_related(field_name)
for field_name in ('levels', ):
if hasattr(mapitemtype, field_name):
queryset.prefetch_related(field_name)
if issubclass(mapitemtype, AreaLocation):
queryset = sorted(queryset, key=AreaLocation.get_sort_key)
if issubclass(mapitemtype, DirectedLineGeometryMapItemWithLevel):
results.extend(obj.to_shadow_geojson() for obj in queryset)
results.extend(obj.to_geojson() for obj in queryset)
return Response(results)
class PackageViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
Retrieve packages the map consists of.
"""
queryset = Package.objects.all()
serializer_class = PackageSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
ordering = ('name',)
class LevelViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
List and retrieve levels.
"""
queryset = Level.objects.all()
serializer_class = LevelSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
ordering = ('altitude',)
class SourceViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
List and retrieve source images (to use as a drafts).
"""
queryset = Source.objects.all()
serializer_class = SourceSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
ordering = ('name',)
include_package_access = True
def get_queryset(self):
return filter_queryset_by_access(self.request, super().get_queryset().all())
@detail_route(methods=['get'])
def image(self, request, name=None):
return self._image(request, name=name, add_cache_key=self._get_add_cache_key(request))
@cache_mapdata_api_response()
def _image(self, request, name=None):
source = self.get_object()
response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0])
image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name)
for chunk in File(open(image_path, 'rb')).chunks():
response.write(chunk)
return response
class LocationViewSet(ViewSet):
"""
List and retrieve locations
"""
# We don't cache this, because it depends on access_list
lookup_field = 'name'
include_package_access = True
@staticmethod
def _filter(queryset):
return queryset.filter(can_search=True).order_by('name')
def list(self, request, **kwargs):
etag = hashlib.sha256(json.dumps({
'full_access': request.c3nav_full_access,
'access_list': request.c3nav_access_list,
'last_update': get_last_mapdata_update().isoformat()
}).encode()).hexdigest()
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
if if_none_match:
if if_none_match == etag:
return HttpResponseNotModified()
locations = []
locations += list(filter_queryset_by_access(request, self._filter(LocationGroup.objects.all())))
locations += sorted(filter_arealocations_by_access(request, self._filter(AreaLocation.objects.all())),
key=AreaLocation.get_sort_key, reverse=True)
response = Response([location.to_location_json() for location in locations])
response['ETag'] = etag
response['Cache-Control'] = 'no-cache'
return response
def retrieve(self, request, name=None, **kwargs):
location = get_location(request, name)
if location is None:
raise Http404
return Response(location.to_json())
@list_route(methods=['POST'])
def wifilocate(self, request):
stations = json.loads(request.POST['stations'])[:200]
if not stations:
return Response({})
bssids = get_bssid_areas_cached()
stations = sorted(stations, key=lambda l: l['level'])
for station in stations:
area_name = bssids.get(station['bssid'])
if area_name is not None:
location = get_location(request, area_name)
if location is not None:
return Response({'location': location.to_location_json()})
return Response({'location': None})

5
src/c3nav/routing/level.py
View file

@ -10,7 +10,6 @@ from scipy.sparse.csgraph._shortest_path import shortest_path
from scipy.sparse.csgraph._tools import csgraph_from_dense
from shapely.geometry import CAP_STYLE, JOIN_STYLE, LineString
from c3nav.access.apply import get_public_packages
from c3nav.mapdata.utils.geometry import assert_multilinestring, assert_multipolygon
from c3nav.mapdata.utils.misc import get_public_private_area
from c3nav.routing.point import GraphPoint
@ -125,13 +124,11 @@ class GraphLevel():
self.rooms.append(room)
def collect_arealocations(self):
public_packages = get_public_packages()
self._built_arealocations = {}
self._built_excludables = {}
for excludable in self.level.arealocations.all():
self._built_arealocations[excludable.name] = excludable.geometry
if excludable.routing_inclusion != 'default' or excludable.package not in public_packages:
if excludable.routing_inclusion != 'default' or not excludable.public:
self._built_excludables[excludable.name] = excludable.geometry
public_area, private_area = get_public_private_area(self.level)

7
src/c3nav/settings.py
View file

@ -3,7 +3,6 @@ import configparser
import os
import string
import sys
from collections import OrderedDict
from django.contrib.messages import constants as messages
from django.utils.crypto import get_random_string
@ -51,12 +50,6 @@ else:
debug_fallback = "runserver" in sys.argv
DEBUG = config.getboolean('django', 'debug', fallback=debug_fallback)
DIRECT_EDITING = config.getboolean('c3nav', 'direct_editing', fallback=DEBUG)
PUBLIC_PACKAGES = [n for n in config.get('c3nav', 'public_packages', fallback='').split(',') if n]
EDITOR_HOSTERS = OrderedDict((name[7:], data) for name, data in config.items() if name.startswith('hoster:'))
OAUTH_CALLBACK_SCHEME = config.get('c3nav', 'oauth_callback_scheme', fallback=None)
OAUTH_CALLBACK_NETLOC = config.get('c3nav', 'oauth_callback_netloc', fallback=None)
RENDER_SCALE = float(config.get('c3nav', 'render_scale', fallback=12.5))
db_backend = config.get('database', 'backend', fallback='sqlite3')