diff --git a/.gitignore b/.gitignore
index 6fd38e15..515126eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ log
dexter/public
.idea
cache
+resources/mysql/places.sql
\ No newline at end of file
diff --git a/assets/css/activity.scss b/assets/css/activity.scss
index a4556430..a2da6ed9 100644
--- a/assets/css/activity.scss
+++ b/assets/css/activity.scss
@@ -34,4 +34,37 @@
.chart.chart-media {
height: 630px;
}
+
+ .chart.chart-media-coverage {
+ height: 300px;
+ }
}
+
+#coverage-map{
+ .feature{
+ fill: #ccc;
+ cursor: pointer;
+ }
+
+ .feature.municipality{
+ opacity: 0;
+ }
+
+ .background {
+ fill: none;
+ pointer-events: all;
+ }
+
+ .feature.active {
+ fill: orange;
+ opacity: 1;
+ }
+
+ .mesh {
+ fill: none;
+ stroke: #fff;
+ stroke-linecap: round;
+ stroke-linejoin: round;
+ }
+
+}
\ No newline at end of file
diff --git a/assets/js/coveragemap.js b/assets/js/coveragemap.js
new file mode 100644
index 00000000..5167ecc3
--- /dev/null
+++ b/assets/js/coveragemap.js
@@ -0,0 +1,290 @@
+(function($, exports) {
+ if (typeof exports.Dexter == 'undefined') exports.Dexter = {};
+ var Dexter = exports.Dexter;
+
+ // view when looking at media coverage
+ Dexter.CoverageView = function() {
+ var self = this;
+
+ // set global variables
+ var form = $(".coverage-refine")
+ var input_selected_province = form.find("input#selected_province")
+ var input_selected_municipality = form.find("input#selected_municipality")
+ var input_published_at = form.find("input#published_at")
+
+ var map_title = $("#coverage-map-title")
+ var map_subtitle = $("#coverage-map-subtitle")
+ var chart_title = $("#medium-breakdown-chart-title")
+ var chart_subtitle = $("#medium-breakdown-chart-subtitle")
+
+ var selected_province = input_selected_province.val()
+ var selected_municipality = input_selected_municipality.val()
+
+ if(selected_province)
+ console.log(selected_province)
+ else
+ console.log("no province selected")
+ if(selected_municipality)
+ console.log(selected_municipality)
+ else
+ console.log("no municipality selected")
+
+ //Width and height
+ var width = 750;
+ var height = 600;
+ var active = d3.select(null);
+
+ var transition_duration = 300;
+
+ //Define map projection
+ var projection = d3.geo.mercator()
+ .translate([width/2, height/2])
+ .center([25.48, -28.76])
+ .scale([2000]);
+
+ //Define path generator
+ var path = d3.geo.path()
+ .projection(projection);
+
+ //Create SVG element
+ var svg = d3.select("#coverage-map")
+ .append("svg")
+ .attr("width", width)
+ .attr("height", height);
+
+ svg.append("rect")
+ .attr("class", "background")
+ .attr("width", width)
+ .attr("height", height)
+ .on("click", reset);
+
+ var g = svg.append("g")
+ .style("stroke-width", "1.5px");
+
+ function pan_and_zoom(d){
+ var bounds = path.bounds(d),
+ dx = bounds[1][0] - bounds[0][0],
+ dy = bounds[1][1] - bounds[0][1],
+ x = (bounds[0][0] + bounds[1][0]) / 2,
+ y = (bounds[0][1] + bounds[1][1]) / 2,
+ scale = .9 / Math.max(dx / width, dy / height),
+ translate = [width / 2 - scale * x, height / 2 - scale * y];
+
+ g.transition()
+ .duration(transition_duration)
+ .style("stroke-width", 1.5 / scale + "px")
+ .attr("transform", "translate(" + translate + ")scale(" + scale + ")");
+ }
+
+ function update_selection(province_id, municipality_id){
+ input_selected_province.val(province_id);
+ input_selected_municipality.val(municipality_id);
+ selected_province = input_selected_province.val()
+ selected_municipality = input_selected_municipality.val()
+ }
+
+ function click_province(d) {
+ if (active.node() === this) return reset();
+ var feature_id = d.id;
+ var feature_name = d.properties.province_name;
+
+ load_municipalities(feature_id);
+
+ active.classed("active", false);
+ active = d3.select(this).classed("active", true);
+
+ map_title.text(feature_name)
+ update_selection(feature_id, null)
+ self.load_and_draw_chart()
+
+ pan_and_zoom(d);
+ }
+
+ function click_municipality(d) {
+ if (active.node() === this) return reset();
+ var feature_id = d.id;
+ var feature_name = d.properties.municipality_name;
+
+ active.classed("active", false);
+ active = d3.select(this).classed("active", true);
+
+ map_title.text(feature_name)
+ update_selection(selected_province, feature_id)
+ self.load_and_draw_chart()
+
+ pan_and_zoom(d);
+ }
+
+ function reset() {
+ active.classed("active", false);
+ active = d3.select(null);
+
+ // reset map title and chart
+ map_title.text("South Africa")
+ update_selection(null, null)
+ self.load_and_draw_chart()
+
+ // remove municipality shapes
+ g.selectAll("path.municipality").remove()
+
+ // zoom out
+ g.transition()
+ .duration(transition_duration)
+ .style("stroke-width", "1.5px")
+ .attr("transform", "");
+ }
+
+ function load_provinces(){
+ d3.json("https://maps.code4sa.org/political/2011/province?filter&quantization=2000", function(error, topo) {
+ if (error){
+ $("#coverage-map").text("No data available.")
+ return
+ }
+
+ //Bind data and create one path per GeoJSON feature
+ g.selectAll("path.province")
+ .data(topojson.feature(topo, topo.objects.demarcation).features)
+ .enter()
+ .append("path")
+ .attr("class", function(d) { return "feature province " + d.id; })
+ .attr("d", path)
+ .on("click", click_province);
+
+ // draw lines around shapes
+ g.append("path")
+ .datum(topojson.mesh(topo, topo.objects.demarcation, function(a, b) { return a !== b; }))
+ .attr("class", "mesh province")
+ .attr("d", path);
+ });
+ }
+
+ function load_municipalities(province_id){
+ d3.json("https://maps.code4sa.org/political/2011/municipality?filter&quantization=1000&filter[province]=" + province_id, function(error, topo) {
+ if (error){
+ $("#coverage-map").text("No data available.")
+ return
+ }
+
+ g.selectAll("path.municipality").remove()
+
+ //Bind data and create one path per GeoJSON feature
+ g.selectAll("path.municipality")
+ .data(topojson.feature(topo, topo.objects.demarcation).features)
+ .enter()
+ .append("path")
+ .attr("class", function(d) { return "feature municipality " + d.id; })
+ .attr("d", path)
+ .on("click", click_municipality);
+
+ // draw lines around shapes
+ g.append("path")
+ .datum(topojson.mesh(topo, topo.objects.demarcation, function(a, b) { return a !== b; }))
+ .attr("class", "mesh municipality")
+ .attr("d", path);
+ });
+ }
+
+ self.init = function() {
+ load_provinces()
+ self.load_and_draw_chart()
+ }
+
+ self.load_and_draw_chart = function(){
+ // update map subtitle
+ var published_at = input_published_at.val()
+ if(published_at)
+ map_subtitle.text(published_at)
+ // load chart data
+ $.getJSON(self.placesUrl(), function(data){
+ console.log(data)
+ if(selected_province)
+ data = data['provinces'][selected_province]
+ if(selected_municipality)
+ data = data['municipalities'][selected_municipality]
+ if(data)
+ {
+ chart_subtitle.text("(" + data.total + " articles)");
+ self.drawChart(data);
+ }
+ else
+ {
+ $(".chart.chart-media-coverage").text("No data available.")
+ chart_subtitle.text("");
+ }
+ });
+ }
+
+ self.placesUrl = function() {
+ var url = document.location;
+
+ if (document.location.search === "") {
+ url = url + "?";
+ } else {
+ url = url + "&";
+ }
+
+ return url + "format=places-json";
+ };
+
+ self.drawChart = function(chart_data) {
+
+ // charts
+ var cats = []
+ var vals = []
+ var medium_breakdown = chart_data.medium_breakdown;
+
+ for (var medium in medium_breakdown) {
+ if (medium_breakdown.hasOwnProperty(medium)) {
+ cats.push(medium);
+ vals.push(medium_breakdown[medium]);
+ }
+ }
+
+ $('.chart-media-coverage').highcharts({
+ chart: {
+ type: 'column'
+ },
+ title: {
+ text: '',
+ style: {
+ display: 'none'
+ }
+ },
+ subtitle: {
+ text: '',
+ style: {
+ display: 'none'
+ }
+ },
+ xAxis: {
+ categories: cats,
+ labels: {
+ step: 1,
+ formatter: function(v) {
+ if (this.value.length > 15) {
+ return this.value.slice(0, 15) + "...";
+ } else {
+ return this.value;
+ }
+ }
+ }
+ },
+ yAxis: {
+ title: {
+ text: '# Articles'
+ }
+ },
+ series: [{
+ showInLegend: false,
+ data: vals
+ }],
+ });
+ }
+ };
+
+})(jQuery, window);
+
+$(function() {
+ var coverage_view = new Dexter.CoverageView
+ coverage_view.init()
+});
diff --git a/assets/js/maps.js b/assets/js/maps.js
index 03bb4bb8..705f7169 100644
--- a/assets/js/maps.js
+++ b/assets/js/maps.js
@@ -14,9 +14,9 @@
self.map.setView({lat: -28.4796, lng: 24.698445}, 5);
var osm = new L.TileLayer('//{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
- minZoom: 1,
- maxZoom: 16,
- attribution: 'Map data © OpenStreetMap contributors'});
+ minZoom: 5,
+ maxZoom: 12,
+ attribution: 'Map data © OpenStreetMap contributors'});
self.map.addLayer(osm);
};
@@ -35,17 +35,17 @@
return url + "format=places-json";
};
-
+
self.loadAndDrawPlaces = function() {
$.getJSON(self.placesUrl(), self.drawPlaces);
};
self.drawPlaceMarker = function(place, coords, radius) {
L.circleMarker(coords, {
- color: 'red',
- fillColor: '#f03',
- fillOpacity: 0.5
- })
+ color: 'red',
+ fillColor: '#f03',
+ fillOpacity: 0.5
+ })
.setRadius(radius)
.addTo(self.map)
.bindPopup(place.full_name + " (" + place.documents.length + ")");
diff --git a/dexter/assets.py b/dexter/assets.py
index 0843e790..7a8a63d4 100644
--- a/dexter/assets.py
+++ b/dexter/assets.py
@@ -67,6 +67,14 @@
'js/activity.js',
output='js/activity.%(version)s.js'))
+assets.register('coveragemap',
+ Bundle(
+ 'js/d3.v3.min.js',
+ 'js/topojson.v1.min.js',
+ 'js/highcharts-4.0.1.js',
+ 'js/coveragemap.js',
+ output='js/activity.%(version)s.js'))
+
assets.register('documents',
Bundle(
maps,
diff --git a/dexter/dashboard.py b/dexter/dashboard.py
index ea7736d0..00672715 100644
--- a/dexter/dashboard.py
+++ b/dexter/dashboard.py
@@ -8,9 +8,9 @@
from flask.ext.login import login_required, current_user
from flask.ext.sqlalchemy import Pagination
from sqlalchemy.sql import func, distinct
-from sqlalchemy.orm import joinedload
+from sqlalchemy.orm import joinedload, subqueryload
-from dexter.models import db, Document, Entity, Medium, User, DocumentSource, DocumentPlace, DocumentFairness, Fairness, Topic
+from dexter.models import db, Document, Entity, Medium, User, DocumentSource, DocumentPlace, DocumentFairness, Fairness, Topic, Place
from dexter.models.document import DocumentAnalysisProblem
from wtforms import validators, HiddenField, TextField, SelectMultipleField
@@ -65,6 +65,92 @@ def monitor_dashboard():
doc_groups=doc_groups)
+@app.route('/coverage-map')
+def coverage_map():
+
+ form = CoverageForm(request.args)
+
+ if form.format.data == 'places-json':
+ # places in json format
+ query = Document.query\
+ .options(joinedload('places').joinedload('place'))
+ query = form.filter_query(query)
+
+ return jsonify(DocumentPlace.summary_for_coverage(query.all()))
+
+ query = Document.query\
+ .options(
+ joinedload(Document.medium),
+ joinedload(Document.topic),
+ joinedload(Document.origin).lazyload('*')
+ )
+ query = form.filter_query(query)
+
+ # do manual pagination
+ query = query.order_by(Document.created_at.desc())
+ document_count = form.filter_query(db.session.query(func.count(distinct(Document.id)))).scalar()
+ paged_docs = query.all()
+
+ return render_template('dashboard/coverage-map.haml',
+ form=form,
+ document_count=document_count,
+ paged_docs=paged_docs)
+
+
+class CoverageForm(Form):
+ medium_id = SelectMultipleField('Medium', [validators.Optional()], default='')
+ published_at = TextField('Published', [validators.Optional()])
+ format = HiddenField('format', default='html')
+ selected_province = HiddenField('selected_province')
+ selected_municipality = HiddenField('selected_municipality')
+
+ def __init__(self, *args, **kwargs):
+ super(CoverageForm, self).__init__(*args, **kwargs)
+
+ self.medium_id.choices = [(str(m.id), m.name) for m in Medium.query.order_by(Medium.name).all()]
+
+ # dynamic default
+ if not self.published_at.data:
+ self.published_at.data = ' - '.join(d.strftime("%Y/%m/%d") for d in [datetime.utcnow() - timedelta(days=14), datetime.utcnow()])
+
+ def media(self):
+ if self.medium_id.data:
+ return Medium.query.filter(Medium.id.in_(self.medium_id.data))
+ else:
+ return None
+
+ @property
+ def published_from(self):
+ if self.published_at.data:
+ return self.published_at.data.split(' - ')[0].strip()
+ else:
+ return None
+
+ @property
+ def published_to(self):
+ if self.published_at.data and ' - ' in self.published_at.data:
+ return self.published_at.data.split(' - ')[1].strip() + ' 23:59:59'
+ else:
+ return self.published_from
+
+ def filter_query(self, query):
+ # Note: this filter is not working as expected
+ # if self.level.data and self.level.data == "province":
+ # query = query.filter(Place.province_code == self.selected_area.data)
+
+ if self.medium_id.data:
+ query = query.filter(Document.medium_id.in_(self.medium_id.data))
+
+ if self.published_from:
+ query = query.filter(Document.published_at >= self.published_from)
+
+ if self.published_to:
+ query = query.filter(Document.published_at <= self.published_to)
+ return query
+
+ def as_dict(self):
+ return dict((f.name, f.data) for f in self if f.name != 'csrf_token')
+
@app.route('/activity')
@login_required
diff --git a/dexter/models/place.py b/dexter/models/place.py
index 39c9eb51..0df463b2 100644
--- a/dexter/models/place.py
+++ b/dexter/models/place.py
@@ -8,7 +8,7 @@
String,
func,
or_,
- )
+)
from sqlalchemy.orm import relationship, backref
import logging
@@ -106,7 +106,7 @@ def as_dict(self):
'code': self.code,
'full_name': self.full_name,
'name': self.name,
- }
+ }
if d['type'] == 'point':
d['coordinates'] = [self.lat, self.lng]
@@ -116,8 +116,8 @@ def as_dict(self):
def __repr__(self):
return "" % (
- self.level, self.province_code, self.municipality_code,
- self.mainplace_name, self.subplace_name)
+ self.level, self.province_code, self.municipality_code,
+ self.mainplace_name, self.subplace_name)
@classmethod
@@ -128,25 +128,25 @@ def find(cls, term):
if term in PLACE_STOPWORDS:
return
- p = Place.query\
- .filter(Place.level == 'province')\
- .filter(Place.province_name == term).first()
+ p = Place.query \
+ .filter(Place.level == 'province') \
+ .filter(Place.province_name == term).first()
if p:
return p
- p = Place.query\
- .filter(Place.level == 'municipality')\
- .filter(or_(
- Place.municipality_name == term,
- Place.municipality_name == 'City of %s' % term)).first()
+ p = Place.query \
+ .filter(Place.level == 'municipality') \
+ .filter(or_(
+ Place.municipality_name == term,
+ Place.municipality_name == 'City of %s' % term)).first()
if p:
return p
- p = Place.query\
- .filter(Place.level == 'mainplace')\
- .filter(or_(
- Place.mainplace_name == term,
- Place.mainplace_name == '%s MP' % term)).first()
+ p = Place.query \
+ .filter(Place.level == 'mainplace') \
+ .filter(or_(
+ Place.mainplace_name == term,
+ Place.mainplace_name == '%s MP' % term)).first()
if p:
return p
@@ -177,7 +177,7 @@ class DocumentPlace(db.Model, WithOffsets):
# offsets in the document, a space-separated list of offset:length pairs.
offset_list = Column(String(1024))
-
+
created_at = Column(DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.current_timestamp())
@@ -223,7 +223,54 @@ def summary_for_docs(cls, docs):
'document_count': count,
'mentions': mentions.values(),
'origins': origins,
- }
+ }
+
+ @classmethod
+ def summary_for_coverage(cls, docs):
+ """
+ Generate a summary description for plotting on a coverage map.
+ """
+
+ report_count = {'total': 0, 'medium_breakdown': {}, 'topic_breakdown': {}, 'provinces': {}}
+
+ for d in docs:
+ # count the total number of articles
+ report_count['total'] += 1
+ # count articles per medium
+ if d.medium:
+ if not report_count['medium_breakdown'].get(d.medium.name):
+ report_count['medium_breakdown'][d.medium.name] = 0
+ report_count['medium_breakdown'][d.medium.name] += 1
+ places = d.get_places()
+ if places:
+ tmp_provinces = {}
+ for dp in places:
+ if not dp.place.province_code in tmp_provinces:
+ tmp_provinces[dp.place.province_code] = []
+ if dp.place.municipality_code:
+ tmp_provinces[dp.place.province_code].append(dp.place.municipality_code)
+ for province_code, municipality_codes in tmp_provinces.iteritems():
+ if not report_count['provinces'].get(province_code):
+ report_count['provinces'][province_code] = {'total': 0, 'medium_breakdown': {}, 'topic_breakdown': {}, 'municipalities': {}}
+ # count the number of articles per province
+ report_count['provinces'][province_code]['total'] += 1
+ # breakdown per medium
+ if d.medium:
+ if not report_count['provinces'][province_code]['medium_breakdown'].get(d.medium.name):
+ report_count['provinces'][province_code]['medium_breakdown'][d.medium.name] = 0
+ report_count['provinces'][province_code]['medium_breakdown'][d.medium.name] += 1
+ for municipality_code in municipality_codes:
+ if not report_count['provinces'][province_code]['municipalities'].get(municipality_code):
+ report_count['provinces'][province_code]['municipalities'][municipality_code] = \
+ {'total': 0, 'medium_breakdown': {}, 'topic_breakdown': {}}
+ # count the number of articles per municipality
+ report_count['provinces'][province_code]['municipalities'][municipality_code]['total'] += 1
+ # breakdown per medium
+ if d.medium:
+ if not report_count['provinces'][province_code]['municipalities'][municipality_code]['medium_breakdown'].get(d.medium.name):
+ report_count['provinces'][province_code]['municipalities'][municipality_code]['medium_breakdown'][d.medium.name] = 0
+ report_count['provinces'][province_code]['municipalities'][municipality_code]['medium_breakdown'][d.medium.name] += 1
+ return report_count
# Places we know aren't in SA, but sometimes match something in our DB
diff --git a/dexter/templates/dashboard/coverage-map.haml b/dexter/templates/dashboard/coverage-map.haml
new file mode 100644
index 00000000..e3f79eb3
--- /dev/null
+++ b/dexter/templates/dashboard/coverage-map.haml
@@ -0,0 +1,60 @@
+%%inherit(file="../layout.haml")
+%%namespace(file="../paginator.haml", **{'import': '*'})
+%%namespace(file="../bootstrap_wtf.haml", **{'import': '*'})
+
+%%block(name='title')
+ Coverage map
+
+%%block(name="extra_header_tags")
+ - for url in webassets('maps-css'):
+ %link(rel="stylesheet", href=url)
+
+
+%%block(name='extra_javascript')
+ - for url in webassets('coveragemap'):
+ %script(src=url)
+
+
+%article#coverage
+
+ .row
+ .col-sm-8
+ %h3
+ Media coverage in
+ %span#coverage-map-title South Africa
+ %nobr %small#coverage-map-subtitle ...
+
+ #tab-map.tab-pane
+ #coverage-map
+
+ .col-sm-4
+ .panel.panel-default#charts
+ .panel-heading
+ %h4.medium-breakdown-chart-title Articles per publisher
+ %nobr %small#medium-breakdown-chart-subtitle ...
+ .panel-body
+ .chart.chart-media-coverage
+ Loading...
+
+ %form.coverage-refine(action=url_for('coverage_map'), method='GET')
+ .panel.panel-default
+ .panel-heading
+ %h3.panel-title
+ Filters
+
+ .panel-body
+ = vertical_field(form.medium_id, class_='select2')
+ .form-group
+ = form.published_at.label()
+ .input-group
+ = form.published_at(class_='form-control use-daterangepicker')
+ %span.input-group-btn
+ %button.btn.btn-default(dataClear='input-group')
+ %i.fa.fa-times
+ .form-group
+ .input-group
+ = form.selected_province
+ = form.selected_municipality
+
+ .panel-footer
+ %input.btn.btn-success(type='Submit', value='Update')
diff --git a/dexter/templates/layout.haml b/dexter/templates/layout.haml
index 59f7e903..913c9dfe 100644
--- a/dexter/templates/layout.haml
+++ b/dexter/templates/layout.haml
@@ -28,25 +28,25 @@
%a.navbar-brand(href="/")
Dexter
- .navbar-nav.navbar-btn
- %a.btn.btn-primary(href=url_for('new_article'))
- %i.fa.fa-plus
- Add article
-
- %ul.nav.navbar-nav
- %li(class_=('active' if request.url_rule.endpoint == 'activity' else ''))
- %a.btn.btn-link(href=url_for('activity'))
- Reports
- %li(class_=('active' if request.url_rule.endpoint == 'monitor_dashboard' else ''))
- %a.btn.btn-link(href=url_for('monitor_dashboard'))
- My Dashboard
-
- - if current_user.admin:
- %li
- %a.btn.btn-link(href='/admin/')
- Dexter Admin
-
- if current_user.is_authenticated():
+ .navbar-nav.navbar-btn
+ %a.btn.btn-primary(href=url_for('new_article'))
+ %i.fa.fa-plus
+ Add article
+
+ %ul.nav.navbar-nav
+ %li(class_=('active' if request.url_rule.endpoint == 'activity' else ''))
+ %a.btn.btn-link(href=url_for('activity'))
+ Reports
+ %li(class_=('active' if request.url_rule.endpoint == 'monitor_dashboard' else ''))
+ %a.btn.btn-link(href=url_for('monitor_dashboard'))
+ My Dashboard
+
+ - if current_user.admin:
+ %li
+ %a.btn.btn-link(href='/admin/')
+ Dexter Admin
+
.navbar-nav.navbar-right
%ul.nav.navbar-nav
%li
@@ -61,10 +61,11 @@
%a(href=url_for('user_logout'), dataMethod='post')
Logout
- %form.navbar-form.navbar-right(action=url_for('search'))
- .form-group
- %input.form-control.allow-enter(type='text', name='q', placeholder='Find people')
- %button.btn.btn-default(type='submit') Go
+ - if current_user.is_authenticated():
+ %form.navbar-form.navbar-right(action=url_for('search'))
+ .form-group
+ %input.form-control.allow-enter(type='text', name='q', placeholder='Find people')
+ %button.btn.btn-default(type='submit') Go
.container
-# show flash messages