TP géoloc addresses

# for starters, for now we only need regular pandas
# we'll go to geopandas a bit later

import pandas as pd

# load the data in data/addresses.csv
# and display a few first lines

addresses = pd.read_csv('data/addresses.csv')
addresses.head(4)

# produce a small extract into data/addresses-small.csv

# your code here

# requests is the swiss knife for doing http

import requests

# to localize ONE address
# we just need to forge the right URL

def localize_one(num, typ, nom):

    # we build the URL which directly contains the address pieces
    url = f"https://data.geopf.fr/geocodage/search/?q={num}+{typ}+{nom},Paris&limit=1"
    
    # print(f"localize_one is fetching page\n{url}")

    # sending request to the web server
    response = requests.get(url)

    # if all is OK, http returns a code in the [200 .. 300[ range
    if not (200 <= response.status_code < 300):
        print(f"WHOOPS.... {response.status_code=}")
        return

    # we can then read the answer 
    # remember it's a JSON string
    # so we can decode it on the fly
    return response.json()

# and here is how we could use it

localize_one(18, 'rue', 'BERNARDINS')

# try to estimate how long it would take 
# to resolve 20_000 addresses this way

# your code here

import requests
# this one is to fake a file from a string
import io

def mass_post(filename, col_number, col_type, col_name, col_city):

    # we don't care about the index for now
    # but note that if we want to merge the result
    # with the initial data, there is a need 
    # for a unique index...

    with open(filename) as feed:
        response = requests.post(
            "https://data.geopf.fr/geocodage/search/csv/", 
            files={'data': feed},
            data={'columns': [col_number, col_type, col_name, col_city]})
    if not (200 <= response.status_code < 300):
        print(f"OOPS, got {response.status_code}")
        return None

    # this time the output is csv, not json
    # note that read_csv requires a file-like object
    # and we have a str, so io.StringIO comes to the rescue
    result = pd.read_csv(io.StringIO(response.text))

    return result


def localize_many(filename, col_number, col_type, col_name, col_city):
    df = pd.read_csv(filename)
    geoloc = mass_post(filename, col_number, col_type, col_name, col_city)
    # because we don't set an index, it is safe to merge
    geoloc = geoloc[['latitude', 'longitude', 'result_city', 'result_type']]
    return df.merge(geoloc, left_index=True, right_index=True)

# try this code on the small sample for starters

addresses_small = localize_many("data/addresses-small.csv", "number", "type", "name", "city")
addresses_small

# sanity check : make sure that all the entries have 
# result_city == 'Paris'
# and
# result_type == 'housenumber'

# your code

# when you think you're ready, go full scale
# be ready to wait for at least 40-60s 

# optional: try to record the time it takes !
# tip: see for example time.time()

# addresses = localize_many("data/addresses.csv", "number", "type", "name", "city")

# sanity check

# len(addresses)

# at this point you should store the data
# it's just good practice, as you've done one important step of the process
# plus it takes quite some time to redo

# so: store geolocalized addresses in data/addresses-geoloc.csv

# your code

import folium

# ~ chatelet

CENTER = 48.856542, 2.347614

# pour commencer on va recharger la dataframe précédente
import pandas as pd

addresses = pd.read_csv("data/addresses-geoloc.csv")

# et en faire un petit échantillon

addresses_small = addresses.iloc[:20]

# créez une map centrée sur ce point et de zoom 13
# n'oubliez pas de l'afficher,
# et vérifiez que vous voyez bien Paris, 
# que vous pouvez zoomer et vous déplacer, ...

# votre code
def paris_map():
    ...

# pour l'afficher
paris_map()

# your code here

def map_addresses(geoloc):
    """
    creates folium map centered on Paris, with the various addresses
    contained in the input dataframe shown as a marker
    """
    pass

# and try it out

# make sure you use A SMALL DATAFRAME because with this method
# trying to display tens of thousands addresses
# is going to be SUPER SLOOOOOW !

map_addresses(addresses_small)

map_small = map_addresses(addresses_small)

# je commente car à ce stade dans le notebook étudiant
# on a map_small is None et donc si on essaie de faire cela
# le notebook plante
# map_small.save("addresses-small.html")

# à ce stade on a besoin de geopandas
# installez-le si besoin

import geopandas as gpd

# ça se lit facilement
quartiers = gpd.read_file("data/quartier_paris.zip", encoding="utf8")

# et le résultat est .. une dataframe
type(quartiers)

# qu'on peut afficher normalement

quartiers

# et on peut même l'afficher sommairement avec matplotlib

# bon c'est loin d'être parfait mais ça nous suffit à ce stade
# on va voir plus loin comment incorporer ça dans la carte

quartiers.plot();

def paris_map():
    """
    create a map of Paris with its 80 quartiers
    """
    map = folium.Map(
        location=CENTER,
        zoom_start=13,
        # width='80%',
    )

    # create a folium JSON object from the geo df
    folium.GeoJson(
        data=quartiers,
        # optionnally we could also tweak things on the way
        # try to uncomment these
        # style_function=lambda x: {"fillColor": "#45a012", "color": "#881212"}
        # the name is in the l_qu column
        # tooltip=folium.GeoJsonTooltip(fields=['l_qu'], aliases=['quartier']),
    # and add it to the map
    ).add_to(map)

    # and that's it
    return map

paris_map()

# in order to display a number in hexadecimal, you can use this 

x = 10
f"{x:02x}"

# your code here

def random_color():
    """
    returns a random color as a string like e.g. #12f285
    that is to say containing 3 bytes in hexadecimal
    """
    # of course this is not the right answer
    return "#12f285"

random_color(), random_color()

# add a random color column in quartiers

# your code here

quartiers.head(3)

# display the quartiers with their individual color

# your code

def paris_map():
    """
    create a map of Paris with its 80 quartiers
    each quartier is shown in its individual color
    """
    pass

# display it
paris_map()

def convert_lat_lon(df):
    """
    the input df is expected to have 2 columns named 'latitude' and 'longitude'

    this function will create a GeoDataFrame based on that, where 'latitude' and 'longitude'
    are replaced by a new column named 'position' 
    also this new column becomes the active geometry for future operations 
    (in our case the spatial join)
    """

    # we need a geopandas-friendly df
    geo_df = gpd.GeoDataFrame(df)

    # beware, here LONGITUDE comes first !
    geo_df['position'] = gpd.points_from_xy(df.longitude, df.latitude)

    # it would make sense to clean up
    # BUT
    # our map-production code relies on these columns, so...
    # del geo_df['latitude']
    # del geo_df['longitude']

    # declare the new column as the active one
    geo_df.set_geometry('position', inplace=True)

    # a technicality: turns out the shapefile was created
    # withthis coordinate reference system
    geo_df.set_crs(epsg=4326, inplace=True)

    return geo_df

# let's apply that to our small input

geoaddresses_small = convert_lat_lon(addresses_small)

# we will also need to set the active column in the quartiers (geo)dataframe

quartiers.set_geometry('geometry', inplace=True)

# spatial join allows to extend the addresses dataframe
# with quartier / arrondissement information

# your code

def add_quartiers(gdf):
    """
    given an addresses geo-dataframe, 
    (i.e. typically produced as an output of convert_lat_lon)
    this function will use spatial join with the quartiers information
    and return a copy of gdf extended with columns such as
    l_qu : name of the quartier
    c_ar : arrondissement number
    color: the (random) color of that quartier
    ...

    """
    # this is not the right answer...
    return gdf

# try your code

# xxx you can safely ignore this warning...
# UserWarning: CRS mismatch between the CRS of left geometries and the CRS of right geometries

geoaddresses_small_extended = add_quartiers(geoaddresses_small)
geoaddresses_small_extended.head(2)

# verify your code

# make sure you have the right number of lines in the result

geoaddresses_small_extended.shape

# rewrite map_addresses so that each address is shown 
# in the color of its quartier

def map_addresses(gdf):
    """
    (slightly) rewrite the first version of this function

    your input is now a geopandas dataframe, with the information
    about the 'quartier'

    and your job is to display all the addresses on the map, now with
    the color of the 'quartier'

    return a folium map of paris with the adresses displayed
    """
    return

# test the new function

map = map_addresses(geoaddresses_small_extended)

map

# votre code

from folium import plugins

def map_addresses(gdf):

    # start like before
    if 'human' not in gdf.columns:
        # create a column with a human-readable address
        gdf['human'] = gdf['number'].astype(str) + ', ' + gdf['type'] + ' ' + gdf['name']

    map = paris_map()


    # we need a JavaScript function...
    # this is required to get  good performances
    callback = """
function (row) {
  // if you need to debug it
  // console.log(row)
  let circle = L.circleMarker(
      // the position
      new L.LatLng(row[0], row[1]),
      // styling
     {color: row[3], radius: 8},
    )
  // add the tooltip
  circle.bindTooltip(row[2]).openTooltip()
  return circle
}
"""

    # compute an extract with fewer columns
    # not strictly necessary but easier to deal with column names
    extract = gdf[['latitude', 'longitude', 'human', 'color']]
    cluster = plugins.FastMarkerCluster(
        extract, callback=callback,
        # nicer behaviour
        disableClusteringAtZoom=18,
    ).add_to(map)

    return map

# let's first test it on the small extract

map_addresses(geoaddresses_small_extended)

# and if all goes well we can try and display the full monty

# first prepare the full dataset
geoaddresses = add_quartiers(convert_lat_lon(addresses))

# and create the map from that
final_map = map_addresses(geoaddresses)

# display the map
final_map

# makes sense to save the hard work

# geoaddresses.to_csv("addresses-final.csv")
# final_map.save("addresses-final.html")