searxng/searx/engines/wikidata.py

import json
from datetime import datetime
from requests import get
from urllib import urlencode

resultCount=2
urlSearch = 'https://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectionsnippet&{query}'
urlDetail = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=labels%7Cinfo%7Csitelinks%7Csitelinks%2Furls%7Cdescriptions%7Cclaims&{query}'
# find the right URL for urlMap
urlMap = 'http://www.openstreetmap.org/?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

def request(query, params):
    params['url'] = urlSearch.format(query=urlencode({'srsearch': query, 'srlimit': resultCount}))
    print params['url']
    return params


def response(resp):
    results = []
    search_res = json.loads(resp.text)
    # TODO parallel http queries
    before = datetime.now()
    for r in search_res.get('query', {}).get('search', {}):
        wikidata_id = r.get('title', '')
        results = results + getDetail(wikidata_id)
    after = datetime.now()
    print str(after - before) + " second(s)"

    return results

def getDetail(wikidata_id):
    language = 'fr'

    url = urlDetail.format(query=urlencode({'ids': wikidata_id, 'languages': language + '|en'}))
    print url
    response = get(url)
    result = json.loads(response.content)
    result = result.get('entities', {}).get(wikidata_id, {})
    
    title = result.get('labels', {}).get(language, {}).get('value', None)
    if title == None:
        title = result.get('labels', {}).get('en', {}).get('value', wikidata_id)
    results = []
    urls = []
    attributes = []

    description = result.get('descriptions', {}).get(language, {}).get('value', '')
    if description == '':
        description = result.get('descriptions', {}).get('en', {}).get('value', '')

    claims = result.get('claims', {})
    official_website = get_string(claims, 'P856', None)
    print official_website
    if official_website != None:
        urls.append({ 'title' : 'Official site', 'url': official_website })
        results.append({ 'title': title, 'url' : official_website })

    if language != 'en':
        add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
    wikipedia_en_link = get_wikilink(result, 'enwiki')
    add_url(urls, 'Wikipedia (en)', wikipedia_en_link)

    if language != 'en':
        add_url(urls, 'Wiki voyage (' + language + ')', get_wikilink(result, language + 'wikivoyage'))
    add_url(urls, 'Wiki voyage (en)', get_wikilink(result, 'enwikivoyage'))

    if language != 'en':
        add_url(urls, 'Wikiquote (' + language + ')', get_wikilink(result, language + 'wikiquote'))
    add_url(urls, 'Wikiquote (en)', get_wikilink(result, 'enwikiquote'))


    add_url(urls, 'Commons wiki', get_wikilink(result, 'commonswiki'))

    add_url(urls, 'Location', get_geolink(claims, 'P625', None))

    add_url(urls, 'Wikidata', 'https://www.wikidata.org/wiki/' + wikidata_id + '?uselang='+ language)

    postal_code = get_string(claims, 'P281', None)
    if postal_code != None:
        attributes.append({'label' : 'Postal code(s)', 'value' : postal_code})

    date_of_birth = get_time(claims, 'P569', None)
    if date_of_birth != None:
        attributes.append({'label' : 'Date of birth', 'value' : date_of_birth})

    date_of_death = get_time(claims, 'P570', None)
    if date_of_death != None:
        attributes.append({'label' : 'Date of death', 'value' : date_of_death})


    results.append({
            'infobox' : title, 
            'id' : wikipedia_en_link,
            'content' : description,
            'attributes' : attributes,
            'urls' : urls
            })

    return results

def add_url(urls, title, url):
    if url != None:
        urls.append({'title' : title, 'url' : url})

def get_mainsnak(claims, propertyName):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return None

    propValue = propValue[0].get('mainsnak', None)
    return propValue

def get_string(claims, propertyName, defaultValue=None):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return defaultValue

    result = []
    for e in propValue:
        mainsnak = e.get('mainsnak', {})

        datatype = mainsnak.get('datatype', '')
        datavalue = mainsnak.get('datavalue', {})
        if datavalue != None:
            result.append(datavalue.get('value', ''))

    if len(result) == 0:
        return defaultValue
    else:
        return ', '.join(result)

def get_time(claims, propertyName, defaultValue=None):
    propValue = claims.get(propertyName, {})
    if len(propValue) == 0:
        return defaultValue

    result = []
    for e in propValue:
        mainsnak = e.get('mainsnak', {})

        datatype = mainsnak.get('datatype', '')
        datavalue = mainsnak.get('datavalue', {})
        if datavalue != None:
            value = datavalue.get('value', '')
            result.append(value.get('time', ''))

    if len(result) == 0:
        return defaultValue
    else:
        return ', '.join(result)

def get_geolink(claims, propertyName, defaultValue=''):
    mainsnak = get_mainsnak(claims, propertyName)

    if mainsnak == None:
        return defaultValue

    datatype = mainsnak.get('datatype', '')
    datavalue = mainsnak.get('datavalue', {})

    if datatype != 'globe-coordinate':
        return defaultValue

    value = datavalue.get('value', {})

    precision = value.get('precision', 0.0002)

    # there is no zoom information, deduce from precision (error prone)    
    # samples :
    # 13 --> 5
    # 1 --> 6
    # 0.016666666666667 --> 9
    # 0.00027777777777778 --> 19
    # wolframalpha : quadratic fit { {13, 5}, {1, 6}, {0.0166666, 9}, {0.0002777777,19}}
    # 14.1186-8.8322 x+0.625447 x^2
    if precision < 0.0003:
        zoom = 19
    else:
        zoom = int(15 - precision*8.8322 + precision*precision*0.625447)

    url = urlMap.replace('{latitude}', str(value.get('latitude',0))).replace('{longitude}', str(value.get('longitude',0))).replace('{zoom}', str(zoom))

    return url

def get_wikilink(result, wikiid):
    url = result.get('sitelinks', {}).get(wikiid, {}).get('url', None)
    if url == None:
        return url
    elif url.startswith('http://'):
        url = url.replace('http://', 'https://')
    elif url.startswith('//'):
        url = 'https:' + url
    return url