// API callback
showrecentposts({"version":"1.0","encoding":"UTF-8","feed":{"xmlns":"http://www.w3.org/2005/Atom","xmlns$openSearch":"http://a9.com/-/spec/opensearchrss/1.0/","xmlns$blogger":"http://schemas.google.com/blogger/2008","xmlns$georss":"http://www.georss.org/georss","xmlns$gd":"http://schemas.google.com/g/2005","xmlns$thr":"http://purl.org/syndication/thread/1.0","id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524"},"updated":{"$t":"2014-10-12T16:30:35.802-04:00"},"category":[{"term":"mathematica"},{"term":"python"},{"term":"Crime Analysis"},{"term":"Chicago Crime"},{"term":"GIS"},{"term":"latex"},{"term":"Autoregressive modeling"},{"term":"Chicago"},{"term":"Chicago Neighborhood Crime"},{"term":"google finance"},{"term":"minute by minute quotes"},{"term":"pythonika"},{"term":"C"},{"term":"Letterhead"},{"term":"MacPorts"},{"term":"Socrata API"},{"term":"Time-series analysis"},{"term":"Wikipedia"},{"term":"backfill stock quotes"},{"term":"dark halos"},{"term":"google insights"},{"term":"latitude"},{"term":"longitude"},{"term":"stock quotes"},{"term":".shp to XML"},{"term":"American"},{"term":"Arduino"},{"term":"Bateman"},{"term":"Batman"},{"term":"Census"},{"term":"Crime"},{"term":"Crime Incident Count"},{"term":"Doctor Who"},{"term":"Google backfill"},{"term":"HTML"},{"term":"Hidden Markov Model"},{"term":"Homicides"},{"term":"MATLAB"},{"term":"Mountain Lion"},{"term":"MySQL"},{"term":"Network Analysis"},{"term":"Number of Wikipedia Edits"},{"term":"OS X"},{"term":"Patrick"},{"term":"Pierce"},{"term":"President"},{"term":"Psycho"},{"term":"Scipy"},{"term":"ScribTeX"},{"term":"Sonic Screwdriver"},{"term":"Statistics"},{"term":"SyntaxHighlighter"},{"term":"TV-B-Gone"},{"term":"TeXShop"},{"term":"The Walking Dead"},{"term":"Universal Remote"},{"term":"Vice"},{"term":"Wikalytics"},{"term":"XML"},{"term":"XML schema"},{"term":"Zombie Apocalypse Modeling"},{"term":"Zombies"},{"term":"blogger"},{"term":"business"},{"term":"card"},{"term":"cards"},{"term":"coin flips"},{"term":"emcee"},{"term":"eye tracking"},{"term":"face tracking"},{"term":"google-code-prettify"},{"term":"head tracking"},{"term":"historical prices"},{"term":"intraday"},{"term":"kaggle"},{"term":"monte carlo"},{"term":"openframeworks"},{"term":"parsing"},{"term":"plotting"},{"term":"probability"},{"term":"pstricks"},{"term":"shp2text"},{"term":"stock trends"},{"term":"template"}],"title":{"type":"text","$t":"Night Time Curiosities"},"subtitle":{"type":"html","$t":"A blog focused on analytics of various subjects using Python and C\/C++, LaTeX tricks, and other neat things."},"link":[{"rel":"http://schemas.google.com/g/2005#feed","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/posts\/default"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default?alt=json-in-script\u0026orderby=published"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/"},{"rel":"hub","href":"http://pubsubhubbub.appspot.com/"},{"rel":"next","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default?alt=json-in-script\u0026start-index=26\u0026max-results=25\u0026orderby=published"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"generator":{"version":"7.00","uri":"http://www.blogger.com","$t":"Blogger"},"openSearch$totalResults":{"$t":"31"},"openSearch$startIndex":{"$t":"1"},"openSearch$itemsPerPage":{"$t":"25"},"entry":[{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-8468884913687297371"},"published":{"$t":"2013-01-29T11:07:00.000-05:00"},"updated":{"$t":"2013-02-05T11:40:30.682-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"MacPorts"},{"scheme":"http://www.blogger.com/atom/ns#","term":"MySQL"},{"scheme":"http://www.blogger.com/atom/ns#","term":"OS X"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"}],"title":{"type":"text","$t":"How to Handle Annoyingly Large CSV Files with MySQL"},"content":{"type":"html","$t":"I recently found some time to look at the data for the \u003Ca href=\"http:\/\/www.kaggle.com\/c\/event-recommendation-engine-challenge\" target=\"_blank\"\u003EEvent Recommendation Engine Challenge\u003C\/a\u003E on \u003Ca href=\"http:\/\/www.kaggle.com\/\" target=\"_blank\"\u003Ekaggle\u003C\/a\u003E. I was very annoyed when I saw one of the files was a 1.19 gig csv. My MacBook Pro only has 4 gigs of ram and most of it is already being used by \u003Ca href=\"http:\/\/www.chromium.org\/\" target=\"_blank\"\u003Echromium\u003C\/a\u003E. In an effort to stymie off the realization that my laptop is five years old, I decided to create a MySQL database from that csv file. The conversion process was a bit trickier than I expected, so I'm writing this post as more of a note to myself.\u003Cbr \/\u003E\u003Cbr \/\u003EFirst make sure \u003Ca href=\"http:\/\/www.macports.org\/\" target=\"_blank\"\u003EMacPorts\u003C\/a\u003E is installed. Then install Python 2.7, MySQL5, the MySQL5 server and the MySQL python library.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: bash; gutter: false;\"\u003Esudo port install python27 mysql5 mysql5-server py27-mysql \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003ECreate the default databases by executing the following command:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: bash; gutter: false;\"\u003Esudo -u mysql mysql_install_db5 \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EStart MySQL (once the server has started, press control+c to escape):\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: bash; gutter: false;\"\u003Esudo \/opt\/local\/lib\/mysql5\/bin\/mysqld_safe \u0026 \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EThen secure the MySQL server with a password:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: bash; gutter: false;\"\u003E\/opt\/local\/lib\/mysql5\/bin\/mysqladmin -u root password [password goes here] \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EHere comes the tricky part. For other languages to access MySQL we will need to create a shortcut to the MySQL socket.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: bash;\"\u003Esudo ln -s \/opt\/local\/var\/run\/mysql5\/mysqld.sock \/tmp\/mysql.sock sudo mkdir \/var\/mysql sudo ln -s \/opt\/local\/var\/run\/mysql5\/mysqld.sock \/var\/mysql\/mysql.sock \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003ENow the fun of converting a large csv file to a SQL database begins. I'm only using python to do the conversion because the algorithm I wrote for the competition was done in python. If any readers are interested, I can write a quick post on converting in other languages. The rules of the competition forbids me from redistributing the data provided for the competition. So let's use a csv file containing the \u003Ca href=\"http:\/\/electoral-vote.com\/evp2013\/Info\/datagalore.html\" target=\"_blank\"\u003Epolling results\u003C\/a\u003E of the 2008 presidential elections for the rest of this blog post. The data looks something like this:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" border=\"1\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003EState\u003C\/th\u003E\u003Cth\u003EObama\u003C\/th\u003E\u003Cth\u003EMcCain\u003C\/th\u003E\u003Cth\u003EStart\u003C\/th\u003E\u003Cth\u003EEnd\u003C\/th\u003E\u003Cth\u003EPollster\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003EAL\u003C\/td\u003E\u003Ctd\u003E36\u003C\/td\u003E\u003Ctd\u003E61\u003C\/td\u003E\u003Ctd\u003E27-Oct\u003C\/td\u003E\u003Ctd\u003E28-Oct\u003C\/td\u003E\u003Ctd\u003ESurveyUSA\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003EAL\u003C\/td\u003E\u003Ctd\u003E34\u003C\/td\u003E\u003Ctd\u003E54\u003C\/td\u003E\u003Ctd\u003E15-Oct\u003C\/td\u003E\u003Ctd\u003E16-Oct\u003C\/td\u003E\u003Ctd\u003ECapital Survey\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003EAL\u003C\/td\u003E\u003Ctd\u003E35\u003C\/td\u003E\u003Ctd\u003E62\u003C\/td\u003E\u003Ctd\u003E8-Oct\u003C\/td\u003E\u003Ctd\u003E9-Oct\u003C\/td\u003E\u003Ctd\u003ESurveyUSA\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003Ctd align=\"center\"\u003E⋮\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EConceptually, the script is very simple. A connection to the MySQL server is made from python. If the database already exists, it is deleted and a new one is created. The header of the csv file is then used to create fields for the new database. Then the data from the csv file is entered into the database and blank entries are converted to 'None.' The last bit tests the new 'polls' database by pulling all of the polling results from Florida. \u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: python;\"\u003Eimport csv import MySQLdb  # open the connection to the MySQL server using MySQLdb mydb = MySQLdb.connect(host='localhost',                        user='root',                        passwd='[your password goes here]'                        ) cursor = mydb.cursor()  # create the database! # if one with a similar name exists, delete it cursor.execute('DROP DATABASE IF EXISTS test') cursor.execute('CREATE DATABASE test') cursor.execute(\"USE test\")   # read the 2008-pres-polls.csv file using the python csv_data = csv.reader(file('2008-pres-polls.csv'))  # read header header = csv_data.next()  # builds fields from the header fields=[] for col in header:     fields.append(('%s VARCHAR(64)' % col)) sqlfields = ',\\n'.join(fields)  # create a table named 'polls' with fields from the header sqltbl = '''CREATE TABLE polls (%s)''' % sqlfields  cursor.execute(sqltbl)  # insert None for empty entries def nullify(d):     def f(x):         if(x == \"\"):             return None         else:             return x     return [f(x) for x in d]  # builds the MySQL data insert command  def buildInsert(table, numfields):     datholder = (numfields-1) * \"%s, \" + \"%s\"     query = (\"insert into %s\" % table) + (\" values (%s)\" % datholder)     return query  # inserts data from each row of csv file numfields = len(header) query = buildInsert('polls', numfields) for line in csv_data:     vals = nullify(line)     cursor.execute(query, vals)  #test database cursor.execute(\"SELECT * FROM polls WHERE State = 'FL' \") results = cursor.fetchall() print results \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EHope this is helpful to someone! You can download all of the files used in this post from \u003Ca href=\"http:\/\/bit.ly\/UA4BqN\" target=\"_blank\"\u003Ehere\u003C\/a\u003E."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/8468884913687297371\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2013\/01\/how-to-handle-annoyingly-large-csv.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8468884913687297371"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8468884913687297371"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2013\/01\/how-to-handle-annoyingly-large-csv.html","title":"How to Handle Annoyingly Large CSV Files with MySQL"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-942195748185838832"},"published":{"$t":"2013-01-23T14:16:00.003-05:00"},"updated":{"$t":"2013-01-23T14:16:58.654-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"dark halos"},{"scheme":"http://www.blogger.com/atom/ns#","term":"emcee"},{"scheme":"http://www.blogger.com/atom/ns#","term":"monte carlo"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"}],"title":{"type":"text","$t":"Kaggling Dark Halos"},"content":{"type":"html","$t":"Success! I scored higher than \u003Ca href=\"http:\/\/www.oamp.fr\/cosmology\/lenstool\/\" target=\"_blank\"\u003ELenstool\u003C\/a\u003E!\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Ca href=\"http:\/\/www.kaggle.com\/c\/DarkWorlds\" target=\"_blank\"\u003EThis\u003C\/a\u003E was one of those Kaggle competitions that I had no right to compete in. My goal was to beat the benchmark, which was a state of the art program in 2007. I will be the first to admit this wasn't the most ambitious goal. However, the closest I have ever been to an astronomy class was an episode of \u003Ca href=\"http:\/\/www.dailymotion.com\/video\/xjpqfn_the-magic-school-bus-1x01-gets-lost-in-space_shortfilms\" target=\"_blank\"\u003EThe Magic School Bus\u003C\/a\u003E - it was the one where they explored all nine planets. Though my best method ranked 121 out of 357, I still consider it a victory (Lenstool ranked 201 out of 357.) My victory definitely exemplifies how childish logic can be successfully used in place of\u0026nbsp;cosmological\u0026nbsp;knowledge.\u003Cbr \/\u003E\u003Ca name='more'\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EI imagined that galaxies would look perfectly circular without any dark matter influence. This also means that one galaxy has no influence over another galaxy. When dark matter is introduced, some force will cause the galaxies to look elongated or elliptical.\u0026nbsp;\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThe python script I wrote to find the coordinates of dark halos reflects this logic. Random points are selected and used to calculate the force of elongation on each galaxy. The difference between the observed ellipticity and the calculated force of elongation is taken. Then the differences are squared and summed. By my childish logic, the lowest calculated value from the selected random points should be where the dark halo is located. For multiple dark halos, the elongation cause by the previously found dark halo(s) should be taken into account before further calculations.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EFor the competition, I created various potentials and regressed for coefficients using the training data. For this blog post, I chose a potential with a double power-law density distribution and randomly picked coefficients. This random potential actually works pretty well as shown below with the data from training skies 1, 101, and 201. The blue circles show where dark halos are located and the red circles are the estimated positions from my python script.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-3ZbSBiJVvic\/UQAtjLOrFpI\/AAAAAAAAAbk\/zwmPsOiMm-s\/s1600\/figure_1.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/1.bp.blogspot.com\/-3ZbSBiJVvic\/UQAtjLOrFpI\/AAAAAAAAAbk\/zwmPsOiMm-s\/s400\/figure_1.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETraining sky 1 with one dark halo\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-mcsACg_4riA\/UQArgzTpn2I\/AAAAAAAAAbQ\/gt66d4OU--Q\/s1600\/figure_101.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/4.bp.blogspot.com\/-mcsACg_4riA\/UQArgzTpn2I\/AAAAAAAAAbQ\/gt66d4OU--Q\/s400\/figure_101.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETraining sky 101 with two dark halos\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-H4d3dBaVO_I\/UQAsBl-_4EI\/AAAAAAAAAbY\/5zFsxhFqoKA\/s1600\/figure_201.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/1.bp.blogspot.com\/-H4d3dBaVO_I\/UQAsBl-_4EI\/AAAAAAAAAbY\/5zFsxhFqoKA\/s400\/figure_201.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETraining sky 201 with three dark halos\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003ETo run the script I have written please download it and the competition's training data from \u003Ca href=\"http:\/\/bit.ly\/10Jvz3a\" target=\"_blank\"\u003Ehere\u003C\/a\u003E and make sure you have\u0026nbsp;\u003Ca href=\"http:\/\/www.python.org\/download\/releases\/2.7\/\" target=\"_blank\"\u003EPython 2.7\u003C\/a\u003E, \u003Ca href=\"http:\/\/dan.iel.fm\/emcee\/\" target=\"_blank\"\u003Eemcee\u003C\/a\u003E, and \u003Ca href=\"http:\/\/www.numpy.org\/\" target=\"_blank\"\u003ENumPy\u003C\/a\u003E\u0026nbsp;installed. Thanks should be given to\u0026nbsp;David Harvey and Thomas Kitching for creating the training data to the\u0026nbsp;\u003Ca href=\"https:\/\/www.kaggle.com\/c\/DarkWorlds\" target=\"_blank\"\u003EObserving Dark Worlds Competition\u003C\/a\u003E. I encourage you to download the data and as Ms. Frizzle says, \"Take chances, make mistakes, and get messy.\"\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/942195748185838832\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2013\/01\/kaggling-dark-halos.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/942195748185838832"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/942195748185838832"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2013\/01\/kaggling-dark-halos.html","title":"Kaggling Dark Halos"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-3ZbSBiJVvic\/UQAtjLOrFpI\/AAAAAAAAAbk\/zwmPsOiMm-s\/s72-c\/figure_1.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-1740137154828688194"},"published":{"$t":"2012-12-04T12:17:00.000-05:00"},"updated":{"$t":"2012-12-04T12:17:07.061-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"dark halos"},{"scheme":"http://www.blogger.com/atom/ns#","term":"kaggle"}],"title":{"type":"text","$t":"Smelling Dark Matter"},"content":{"type":"html","$t":"I've been using the little free time I have to compete in kaggle's\u0026nbsp;\u003Ca href=\"http:\/\/www.kaggle.com\/c\/DarkWorlds\" target=\"_blank\"\u003EObserving Dark Worlds\u003C\/a\u003E\u0026nbsp;competition. I have to say that this machine learning competition has given me a new appreciation of what astronomers deal with in their research. It also illustrated why so many of the\u0026nbsp;astronomy\u0026nbsp;majors I knew in college always looked so sleep deprived - I had attributed it to the nocturnal nature of twinkles.\u0026nbsp;\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EIn the Observing Dark Worlds competition, the goal is to pin point the locations of invisible dark matter halos with only the coordinates of galaxies and their respective\u0026nbsp;ellipticity. Below is a quick visual example of the difficulty of this task (my code for the visualizations can be found \u003Ca href=\"http:\/\/www.kaggle.com\/c\/DarkWorlds\/forums\/t\/3197\/code-for-plotting-the-galaxies-in-python\" target=\"_blank\"\u003Ehere\u003C\/a\u003E).\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-RPUD3CcjbHs\/UL4hA38Fe8I\/AAAAAAAAAaA\/JB48VUAZC44\/s1600\/sky150_figure_3.png\" imageanchor=\"1\" style=\"clear: left; margin-bottom: 1em; margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/1.bp.blogspot.com\/-RPUD3CcjbHs\/UL4hA38Fe8I\/AAAAAAAAAaA\/JB48VUAZC44\/s320\/sky150_figure_3.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EThis is the visualization of the data from training sky 150.\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-u-9Hbxw3c-U\/UL4g9NF6AXI\/AAAAAAAAAZ4\/AIn1VzVUwzo\/s1600\/sky150_figure_4.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/2.bp.blogspot.com\/-u-9Hbxw3c-U\/UL4g9NF6AXI\/AAAAAAAAAZ4\/AIn1VzVUwzo\/s320\/sky150_figure_4.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EThe two blue circles are the dark halos hiding in training sky 150.\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EI haven't made much progress in my analysis - in fact I have never been this stumped on any problem. So far I have only been able to determine the location of one dark matter halo exactly and generate a few probable locations of the other one. Below is the visualization of my progress for training sky 150 and training sky 124 from the competition. The red circles are my estimated locations and the blue circles are the true locations of the dark matter halos.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-YH2c1OEhUsc\/UL4pgfZ453I\/AAAAAAAAAac\/G2p8YkpC2SA\/s1600\/sky150_figure_2.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/1.bp.blogspot.com\/-YH2c1OEhUsc\/UL4pgfZ453I\/AAAAAAAAAac\/G2p8YkpC2SA\/s320\/sky150_figure_2.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETraining Sky 150\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-FSO-wbFymyg\/UL4qDAWleDI\/AAAAAAAAAak\/D416WEFHDZs\/s1600\/sky124_figure_1.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/4.bp.blogspot.com\/-FSO-wbFymyg\/UL4qDAWleDI\/AAAAAAAAAak\/D416WEFHDZs\/s320\/sky124_figure_1.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETraining Sky 124\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EIf it is before the competition deadline and you are interested in my method or code, I am more than happy to win this competition through a team effort! For my regular readers, I will make a detailed posting after the competition deadline.\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/1740137154828688194\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/12\/smelling-dark-matter.html#comment-form","title":"1 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1740137154828688194"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1740137154828688194"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/12\/smelling-dark-matter.html","title":"Smelling Dark Matter"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-RPUD3CcjbHs\/UL4hA38Fe8I\/AAAAAAAAAaA\/JB48VUAZC44\/s72-c\/sky150_figure_3.png","height":"72","width":"72"},"thr$total":{"$t":"1"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-8817843853512925808"},"published":{"$t":"2012-09-04T14:02:00.001-04:00"},"updated":{"$t":"2012-09-04T14:07:44.638-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"coin flips"},{"scheme":"http://www.blogger.com/atom/ns#","term":"probability"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"}],"title":{"type":"text","$t":"What is the Probability of Getting (k) Heads in a Row for (n) Consecutive Tosses?"},"content":{"type":"html","$t":"I asked myself a fun question after reading a \u003Ca href=\"https:\/\/www.quantnet.com\/threads\/choosing-tt-vs-th-problem.10914\/\" target=\"_blank\"\u003Epost\u003C\/a\u003E on QuantNet. What are the odds of getting two, four, or six heads after five, ten, or a hundred consecutive tosses of a fair coin? It seemed like a fun high school leveled math problem and with some quick python I was able to generate a pretty graph to answer this question.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-d4QAdP4ef0g\/UEY0asFsqhI\/AAAAAAAAAY4\/KLywFvVENKQ\/s1600\/OddsofHeadsinaRow.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/3.bp.blogspot.com\/-d4QAdP4ef0g\/UEY0asFsqhI\/AAAAAAAAAY4\/KLywFvVENKQ\/s400\/OddsofHeadsinaRow.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Ca name='more'\u003E\u003C\/a\u003EMy approach to this problem is actually quite brutish because of the tricky counting. You cannot assume that the probability at each step in the sequence is identical. Even though the inherent probability of the fair coin is still 0.5, the sequences are not completely\u0026nbsp;independent due to causality. Here is a quick demonstration for counting two heads out of five tosses to illustrate this point.\u003Cbr \/\u003E\u003Ctable\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd\u003E\u003C\/td\u003E\u003Cth colspan=\"2\"\u003E\u003Cbr \/\u003E\u003Ch3\u003EPossible Outcomes\u003C\/h3\u003E\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd valign=\"top\"\u003EToss 1\u003C\/td\u003E\u003Ctd align=\"center\"\u003EH\u003C\/td\u003E\u003Ctd align=\"center\"\u003ET\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd valign=\"top\"\u003EToss 2\u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cspan style=\"background-color: yellow;\"\u003EHH\u003C\/span\u003E\u003Cbr \/\u003ETH\u003C\/td\u003E\u003Ctd align=\"center\"\u003EHT\u003Cbr \/\u003ETT\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd valign=\"top\"\u003EToss 3\u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHH\u003C\/del\u003E\u003Cbr \/\u003EHTH\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003ETHH\u003C\/span\u003E\u003Cbr \/\u003ETTH\u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHT\u003C\/del\u003E\u003Cbr \/\u003EHTT\u003Cbr \/\u003ETHT\u003Cbr \/\u003ETTT \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd valign=\"top\"\u003EToss 4\u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHHH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003EHTHH\u003C\/span\u003E\u003Cbr \/\u003EHTTH\u003Cbr \/\u003E\u003Cdel\u003ETHHH\u003C\/del\u003E\u003Cbr \/\u003ETHTH\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003ETTHH\u003C\/span\u003E\u003Cbr \/\u003ETTTH \u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTT\u003C\/del\u003E\u003Cbr \/\u003EHTHT\u003Cbr \/\u003EHTTT\u003Cbr \/\u003E\u003Cdel\u003ETHHT\u003C\/del\u003E\u003Cbr \/\u003ETHTT\u003Cbr \/\u003ETTHT\u003Cbr \/\u003ETTTT \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd valign=\"top\"\u003EToss 5\u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHHHH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHHTH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHTHHH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHHTT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTTH\u003C\/del\u003E\u003Cbr \/\u003EHTHTH\u003Cbr \/\u003E\u003Cdel\u003ETHHHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003ETHTHH\u003C\/span\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTTT\u003C\/del\u003E\u003Cbr \/\u003EHTTHT\u003Cbr \/\u003E\u003Cdel\u003ETHHTT\u003C\/del\u003E\u003Cbr \/\u003ETHTTH\u003Cbr \/\u003ETTHTH\u003Cbr \/\u003EHTTTT\u003Cbr \/\u003ETTHTT\u003Cbr \/\u003ETTTTH \u003C\/td\u003E\u003Ctd align=\"center\"\u003E\u003Cdel\u003EHHHHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTHH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003ETHHHH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHHTHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003EHTHHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003EHTTHH\u003C\/span\u003E\u003Cbr \/\u003E\u003Cdel\u003ETHHTH\u003C\/del\u003E\u003Cbr \/\u003E\u003Cdel\u003ETTHHH\u003C\/del\u003E\u003Cbr \/\u003EHTHTT\u003Cbr \/\u003EHTTTH\u003Cbr \/\u003ETHTHT\u003Cbr \/\u003E\u003Cdel\u003ETTHHT\u003C\/del\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: yellow;\"\u003ETTTHH\u003C\/span\u003E\u003Cbr \/\u003ETHTTT\u003Cbr \/\u003ETTTHT\u003Cbr \/\u003ETTTTT \u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003EThe strikethroughs are the outcomes that have been eliminated because they have previously yielded to heads in a row. The highlighted sequences are the outcomes that can successfully produce two heads on that toss. Using the example count for a sequence of two heads out of three tosses, on the third toss there is one possibility of producing a sequence with two consecutive heads (instead of three) and there are six possible outcomes (instead of eight). The differences derives from the first possible successful sequencing of two heads in a row from the second toss.\u003Cbr \/\u003E\u003Cbr \/\u003EThis counting was performed for the sequences of three and four heads in a row. The results are summarized in the tables below, along with the sequence of two heads in a row.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" border=\"1\"\u003E  \u003Ccaption\u003E Two Heads in a Row\u003C\/caption\u003E  \u003Ctbody\u003E\u003Ctr\u003E       \u003Cth scope=\"col\"\u003E\u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 1 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 2 \u003C\/th\u003E     \u003Cth scope=\"col\"\u003EToss 3 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 4 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 5 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 6 \u003C\/th\u003E   \u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003ESuccesses \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E3 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E5 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EFailures \u003C\/th\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E3 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E5 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E8 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E13 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E21 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EOutcomes \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E6 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E10 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E16 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E26 \u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" border=\"1\"\u003E  \u003Ccaption\u003E Three Heads in a Row\u003C\/caption\u003E  \u003Ctbody\u003E\u003Ctr\u003E       \u003Cth scope=\"col\"\u003E\u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 1 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 2 \u003C\/th\u003E     \u003Cth scope=\"col\"\u003EToss 3 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 4 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 5 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 6 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 7 \u003C\/th\u003E   \u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003ESuccesses \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E7 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EFailures \u003C\/th\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E7 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E13 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E24 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E44 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E81 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EOutcomes \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E8 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E14 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E26 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E48 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E88 \u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" border=\"1\"\u003E  \u003Ccaption\u003E Four Heads in a Row\u003C\/caption\u003E  \u003Ctbody\u003E\u003Ctr\u003E       \u003Cth scope=\"col\"\u003E\u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 1 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 2 \u003C\/th\u003E     \u003Cth scope=\"col\"\u003EToss 3 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 4 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 5 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 6 \u003C\/th\u003E       \u003Cth scope=\"col\"\u003EToss 7 \u003C\/th\u003E     \u003Cth scope=\"col\"\u003EToss 8 \u003C\/th\u003E   \u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003ESuccesses \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E0 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E1 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E8 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EFailures \u003C\/th\u003E    \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E8 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E15 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E29 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E56 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E108 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E208 \u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E      \u003Cth scope=\"row\"\u003EOutcomes \u003C\/th\u003E      \u003Ctd align=\"center\"\u003E2 \u003C\/td\u003E      \u003Ctd align=\"center\"\u003E4 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E8 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E16 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E30 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E58 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E112 \u003C\/td\u003E    \u003Ctd align=\"center\"\u003E216 \u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003EHere is the elegance that I miss about high school mathematics. You have probably already noticed that the successes, the failures, and the outcomes are expressible as \u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Generalizations_of_Fibonacci_numbers\" target=\"_blank\"\u003Egeneralized fibonacci numbers\u003C\/a\u003E. The successes for the sequence involving two heads in a row are the well-known Fibonacci number. The successes for the sequence involving three heads in a row are the lesser known tribonacci numbers. The\u0026nbsp;successes for the sequence involving three heads in a row are the almost unknown tetranacci numbers. These high order sequences can be recursively expressed with n as number of tosses, k as the number of heads in a row, and for k \u0026gt; 2 as:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[Fib_{k}=F_{k}^{(n)}=\\sum_{i=1}^{n} F_{k-i}^{(n)}\\]\u003Cbr \/\u003EThen to calculate the probability of success for k heads in a row at the n\u003Ci\u003E\u003Cspan style=\"font-size: x-small;\"\u003Eth\u003C\/span\u003E\u003C\/i\u003E toss would be:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[probability_n=1-\\frac{Fib_{k}(n+k)}{2 \\times\u0026nbsp;Fib_{k}(n-1+k)}\\]\u003Cbr \/\u003ETo calculate the total probability of seeing k heads in a row after n tosses would be:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[probability=1-\\frac{Fib_{k}(1+k)}{2 \\times\u0026nbsp;Fib_{k}(1-1+k)}\u0026nbsp;\\times\u0026nbsp;\\frac{Fib_{k}(2+k)}{2 \\times\u0026nbsp;Fib_{k}(2-1+k)}\u0026nbsp;\\times\u0026nbsp;\\frac{Fib_{k}(3+k)}{2 \\times\u0026nbsp;Fib_{k}(3-1+k)} \\ldots \\times\u0026nbsp;\\frac{Fib_{k}(n+k)}{2 \\times\u0026nbsp;Fib_{k}(n-1+k)}\\]\u003Cbr \/\u003EThis result is readily translated into python to produce the pretty graph at the beginning of the post. The code is presented below:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: python;\"\u003Efrom decimal import Decimal  # function for generalized fibonacci sequences def Fib(n,k):     if n \u0026lt; k-1:         return 0     elif n == k-1:         return 1     else:         f =[0] * (n+1)         for i in xrange(k-1):             f[i] = 0         f[k-1] = 1         for i in xrange(k,n+1):             a = 0             for j in xrange(k+1):                 a += f[i-j]             f[i] = a         return f[n]  # function to calculate failures\/outcomes def ProbHnFail(n,k):     b = 2*Fib(n-1+k,k) # number of outcomes     if n \u0026lt; k:         a = b     else:         a = Fib(n+k,k) # number of failures         if a == 0:             a = b     p = (Decimal(a)\/Decimal(b))     return p  # calculates and plots probabilities import numpy as np  Tosses = 100 t1 = np.arange(1, Tosses+1, 1)  H2=[] for x in t1:     y = ProbHnFail(x,2)     H2.append(y) a = 1-np.cumprod(H2)  H3=[] for x in t1:     y = ProbHnFail(x,4)     H3.append(y) b = 1-np.cumprod(H3)  H4=[] for x in t1:     y = ProbHnFail(x,6)     H4.append(y) c = 1-np.cumprod(H4)  import matplotlib.pyplot as plt  fig = plt.figure() ax = fig.add_subplot(111) ax.plot(t1,a, t1,b, t1,c) ax.set_ylim([0,1.01]) ax.grid(True) ax.legend(('HH','HHHH','HHHHHH'), 'best') ax.set_xlabel('Number of Consecutive Tosses') ax.set_ylabel('Probability of Appearance') ax.set_title('Probability after n-Tosses') plt.show() \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EHappy\u0026nbsp;flipping!"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/8817843853512925808\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/09\/what-is-probability-of-getting-k-heads.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8817843853512925808"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8817843853512925808"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/09\/what-is-probability-of-getting-k-heads.html","title":"What is the Probability of Getting (k) Heads in a Row for (n) Consecutive Tosses?"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/3.bp.blogspot.com\/-d4QAdP4ef0g\/UEY0asFsqhI\/AAAAAAAAAY4\/KLywFvVENKQ\/s72-c\/OddsofHeadsinaRow.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-4006632449240212398"},"published":{"$t":"2012-08-30T14:48:00.002-04:00"},"updated":{"$t":"2012-12-12T14:38:03.498-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"latex"}],"title":{"type":"text","$t":"LaTeX Template for Lecture or Class Notes"},"content":{"type":"html","$t":"A friend of mine asked me to create a LaTeX template for lecture notes. He wanted to learn LaTeX by typesetting all of his class notes. I think I created a pretty nice template and wanted to share it with everyone.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ciframe src=\"http:\/\/docs.google.com\/viewer?url=https%3A%2F%2Fdl.dropbox.com%2Fu%2F5311161%2FBlogSource%2Fpdfs%2FLecture_Notes-example.pdf\u0026embedded=true\" width=\"600\" height=\"780\" style=\"border: none;\"\u003E\u003C\/iframe\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EYou can download the template from \u003Ca href=\"http:\/\/bit.ly\/N1Pp3k\" target=\"_blank\"\u003Ehere\u003C\/a\u003E! Happy TeXing!"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/4006632449240212398\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/latex-template-for-lecture-or-class.html#comment-form","title":"2 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/4006632449240212398"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/4006632449240212398"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/latex-template-for-lecture-or-class.html","title":"LaTeX Template for Lecture or Class Notes"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"thr$total":{"$t":"2"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-3906800063701973633"},"published":{"$t":"2012-08-28T13:54:00.001-04:00"},"updated":{"$t":"2012-08-28T13:58:03.476-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"latex"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"TeXShop"}],"title":{"type":"text","$t":"Syntax Highlighting in LaTeX with Minted"},"content":{"type":"html","$t":"Here is a quick tip for those of you who use MacPorts and want pretty syntax highlighting in your LaTeX documents. Recently I discovered a great looking package called \u003Ca href=\"http:\/\/www.ctan.org\/pkg\/minted\" target=\"_blank\"\u003Eminted\u003C\/a\u003E, which depends on an external python syntax highlighter called \u003Ca href=\"http:\/\/pygments.org\/\" target=\"_blank\"\u003EPygments\u003C\/a\u003E. I couldn't find any decent tutorials online for a quick and easy setup, so I decided to write one. I hope it will be helpful for someone.\u003Cbr \/\u003E\u003Cbr \/\u003ETo my readers who didn't come upon this article through google, the minted package is a better alternative to the listings package. Minted allows you to present code in techno-color! This means the programs you have worked hard on, like:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: c\"\u003Eint main() {  printf(\"hello, world\");  return 0; } \u003C\/syntaxhighlighter\u003Ewill no longer be restricted to marginalizing black and white:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-2-eFCdyW-G0\/UDz-9l3QNDI\/AAAAAAAAAYA\/onTxS6KAWTM\/s1600\/minimal-listings.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"67\" src=\"http:\/\/2.bp.blogspot.com\/-2-eFCdyW-G0\/UDz-9l3QNDI\/AAAAAAAAAYA\/onTxS6KAWTM\/s200\/minimal-listings.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EWith minted your code will finally be noticed in the appendix of that report...\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-bMQj1S74oL4\/UDz_DFlj5JI\/AAAAAAAAAYI\/yUM4WQ8G3ys\/s1600\/minimal.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"71\" src=\"http:\/\/3.bp.blogspot.com\/-bMQj1S74oL4\/UDz_DFlj5JI\/AAAAAAAAAYI\/yUM4WQ8G3ys\/s200\/minimal.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cspan style=\"font-size: xx-small;\"\u003EDisclaimer: Minted can do many things, but it will not make bad programs work. Though it will make them look like they could work.\u003C\/span\u003E\u003Cbr \/\u003E\u003Ca name='more'\u003E\u003C\/a\u003E\u003Cbr \/\u003E\u003Ch3\u003EThe Setup\u003C\/h3\u003E1. Make sure you have \u003Ca href=\"http:\/\/www.macports.org\/\" target=\"_blank\"\u003EMacPorts\u003C\/a\u003E\u0026nbsp;and \u003Ca href=\"http:\/\/tug.org\/mactex\/\" target=\"_blank\"\u003ELaTeX\u003C\/a\u003E installed.\u003Cbr \/\u003E\u003Cbr \/\u003E2. Install Python. If you don't have it installed already, here is an example of how you would install Python 2.7 with MacPorts in the terminal:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: shell\"\u003Esudo port install python27\u0026nbsp;\u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003E3. Install the proper Pygments package. For continuity, I'm going to show how to do this with the version of Python installed in step 2. In the terminal type the following:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: shell\"\u003Esudo port install py27-pygments\u0026nbsp;\u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003E4. Finally, create a soft link from the Pygments package to your \/usr\/local\/bin folder and the setup is complete.\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: shell\"\u003Esudo ln -s \/opt\/local\/Library\/Frameworks\/Python.framework\/Versions\/2.7\/bin\/pygmentize \/usr\/local\/bin\/pygmentize\u0026nbsp;\u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003E\u003Ch3\u003EThe Usage\u003C\/h3\u003ELet's run some examples! One of the basic examples provided in the \u003Ca href=\"http:\/\/mirror.jmu.edu\/pub\/CTAN\/macros\/latex\/contrib\/minted\/minted.pdf\" target=\"_blank\"\u003Eminted documentation\u003C\/a\u003E is called minimal.tex. If you are feeling lazy, you can download it from \u003Ca href=\"http:\/\/bit.ly\/PYJr2P\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. The TeX file contains the first program everyone writes in C:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: latex\"\u003E\\documentclass{article}  \\usepackage{minted}  \\begin{documents} \\begin{minted}[linenos]{c} int main() {  printf(\"hello, world\");  return 0; } \\end{minted} \\end{document} \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EThe minted documentation assumes that you are compiling minimal.tex in the terminal. If this is true, to compile minimal.tex you need only to send this command:\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: shell\"\u003Epdflatex -shell-escape minimal \u003C\/syntaxhighlighter\u003EI usually like to use \u003Ca href=\"http:\/\/pages.uoregon.edu\/koch\/texshop\/\" target=\"_blank\"\u003ETeXShop\u003C\/a\u003E\u0026nbsp;for my LaTeX needs. For us TeXShoppers, we must do some tinkering to mimic the \"-shell-escape\" command that is needed to compile the minted package. Go to the TeXShop Preferences, choose the Engine tab, and type \"-shell-escape\" under the pdfTeX\/Latex section.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-BOD3cXw7SY4\/UD0DiQgA4JI\/AAAAAAAAAYg\/ijIrHoOV4TI\/s1600\/Screen.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"400\" src=\"http:\/\/4.bp.blogspot.com\/-BOD3cXw7SY4\/UD0DiQgA4JI\/AAAAAAAAAYg\/ijIrHoOV4TI\/s400\/Screen.png\" width=\"319\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ENow typeset it! Both methods should result in a PDF containing:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-bMQj1S74oL4\/UDz_DFlj5JI\/AAAAAAAAAYI\/yUM4WQ8G3ys\/s1600\/minimal.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"115\" src=\"http:\/\/3.bp.blogspot.com\/-bMQj1S74oL4\/UDz_DFlj5JI\/AAAAAAAAAYI\/yUM4WQ8G3ys\/s320\/minimal.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/3906800063701973633\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/syntax-highlighting-in-latex-with-minted.html#comment-form","title":"3 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3906800063701973633"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3906800063701973633"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/syntax-highlighting-in-latex-with-minted.html","title":"Syntax Highlighting in LaTeX with Minted"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-2-eFCdyW-G0\/UDz-9l3QNDI\/AAAAAAAAAYA\/onTxS6KAWTM\/s72-c\/minimal-listings.png","height":"72","width":"72"},"thr$total":{"$t":"3"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-174396202267129343"},"published":{"$t":"2012-08-22T15:12:00.001-04:00"},"updated":{"$t":"2012-08-22T15:12:33.715-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Number of Wikipedia Edits"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Wikalytics"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Wikipedia"}],"title":{"type":"text","$t":"How to Count the Number of Wikipedia Edits"},"content":{"type":"html","$t":"This week I finally found some time to catch up on The Colbert Report and I discovered a\u0026nbsp;mischievous little gem. Stephen Colbert and his staff brought up an old Washington Post article, \"\u003Ca href=\"http:\/\/www.washingtonpost.com\/wp-dyn\/content\/article\/2008\/08\/29\/AR2008082902691.html\" target=\"_blank\"\u003EWikipedia Edits Forecast Vice Presidential Picks\u003C\/a\u003E\" and subtly suggested that editing is like voting - only with no photo id checks. From five minutes of Googling, I found a torrent of news articles with litanies of what I summarized in the previous sentence. Then I discovered \u003Ca href=\"http:\/\/techpresident.com\/news\/22680\/how-spot-romneys-vice-president-pick-advance?utm_source=feedburner\u0026amp;utm_medium=feed\u0026amp;utm_campaign=Feed:+techpres+%28techPresident%29\" target=\"_blank\"\u003Ethis article\u003C\/a\u003E that somewhat substantiates the claim with an ugly excel chart, below.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/techpresident.com\/files\/Screen%20shot%202012-08-06%20at%2012.40.52%20PM_0.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"220\" src=\"http:\/\/techpresident.com\/files\/Screen%20shot%202012-08-06%20at%2012.40.52%20PM_0.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ETo remind those who have forgotten, Mitt Romney announced Paul Ryan as his running mate on August 11, 2012.\u003Cbr \/\u003E\u003Cbr \/\u003EIn all the articles I read on this claim, I was surprised that almost no news source visualized the data or told readers how to gather the data. I've previously \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2012\/04\/can-i-trust-wikipedia.html\" target=\"_blank\"\u003Eposted\u003C\/a\u003E on some fun analytics with Wikipedia using Mathematica. However, with my recent clean installation of Mountain Lion, it has become clear to me that not everyone has access to Mathematica. In this post, I will show how to count the number of revisions\/edits on a Wikipedia article using delightfully free python.\u003Cbr \/\u003E\u003Cbr \/\u003EThe python script I have written is quite easy to understand and depends on the Numpy external library. The script starts out by accessing some raw revisions\/edits data for a certain article.\u0026nbsp;To get the revision histories for a Wikipedia article there are two methods. One way is to scrap the html, which can take some effort. My code uses the second by simply calling the \u003Ca href=\"http:\/\/www.mediawiki.org\/wiki\/MediaWiki\" target=\"_blank\"\u003EMediaWiki\u003C\/a\u003E API. The revision data requested through the API is in an XML format. This data is read into python in an XML format and then parsed for only the date of the revision. The dates are then tallied by a tallying function I have written and sorted by date. Finally the revision\/edits data is outputted by the python script as a properly formated array, just waiting to be plotted. The commented python code is presented below: \u003Csyntaxhighlighter class=\"brush: python;\"\u003Eimport urllib2 import xml.etree.ElementTree as ET import dateutil.parser from collections import Counter import numpy as np  def tally(datalist):     lst = Counter(datalist)     a = []     for i,j in lst.items():         a.append([i,j])     return a  def revisions(articlename):     articlename = articlename.replace(' ','_') # format article title for url     url = 'http:\/\/en.wikipedia.org\/w\/api.php?action=query\u0026prop=revisions\u0026titles=%s\u0026' % articlename + \\     'rvprop=user|timestamp\u0026rvlimit=max\u0026redirects$rvuser\u0026format=xml' # data limit is set to max for 500 edits     req = urllib2.urlopen(url)     data = req.read(); req.close() # reads url response     root = ET.fromstring(data) # reads xml data     group = root.find(\".\/\/revisions\")     results = []     ## gets revision times from xml data     for elem in group.getiterator('rev'):         timestamp = elem.get('timestamp')         timestamp = dateutil.parser.parse(timestamp).date() # parses timestamp and returns only date         results.append(timestamp)     a = tally(results) # tallys by date     datetally = np.array(sorted(a, key=lambda x: x[0])) # sorts tally by date     return datetally \u003C\/syntaxhighlighter\u003EHere is a quick example of how to plot with the array that is returned. I chose to use Tim Pawlenty and Marco Rubio to show the limitations of the MediaWiki API. I am also biased towards Pawlenty because of his \u003Ca href=\"http:\/\/www.youtube.com\/watch?v=YfkNEq1XioE\" target=\"_blank\"\u003Eamazing ads\u003C\/a\u003E during the GOP primaries. There are Wikipedia pages that have low daily revision activity for long stretches of time and pages with high amounts of revisions in very short periods.\u0026nbsp;The MediaWiki API will return only the previous 500 revisions on any article unless you have a super user status.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-sNR4e3qgcsY\/UDUpHS8chGI\/AAAAAAAAAXo\/UXFEzq5wbwA\/s1600\/wikipediaedits.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/2.bp.blogspot.com\/-sNR4e3qgcsY\/UDUpHS8chGI\/AAAAAAAAAXo\/UXFEzq5wbwA\/s400\/wikipediaedits.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: python;\"\u003Efrom matplotlib import pyplot as plt  a = revisions('Tim Pawlenty') b = revisions('Marco Rubio')  fig = plt.figure()  graph = fig.add_subplot(111) graph.plot(a[:,0], a[:,1], 'r', b[:,0], b[:,1], 'b') fig.autofmt_xdate() plt.legend(('Tim Pawlenty', 'Marco Rubio'), loc='upper left') plt.title('Number of Wikipedia Edits') plt.show() \u003C\/syntaxhighlighter\u003EMy warning for this script, please use some common sense to interpret results. You can download the source \u003Ca href=\"http:\/\/bit.ly\/R0Rd9c\" target=\"_blank\"\u003Ehere\u003C\/a\u003E."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/174396202267129343\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/how-to-count-number-of-wikipedia-edits.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/174396202267129343"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/174396202267129343"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/how-to-count-number-of-wikipedia-edits.html","title":"How to Count the Number of Wikipedia Edits"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-sNR4e3qgcsY\/UDUpHS8chGI\/AAAAAAAAAXo\/UXFEzq5wbwA\/s72-c\/wikipediaedits.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-2485814839060809259"},"published":{"$t":"2012-08-15T13:30:00.001-04:00"},"updated":{"$t":"2012-09-11T13:22:48.284-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Google backfill"},{"scheme":"http://www.blogger.com/atom/ns#","term":"google finance"},{"scheme":"http://www.blogger.com/atom/ns#","term":"historical prices"},{"scheme":"http://www.blogger.com/atom/ns#","term":"intraday"},{"scheme":"http://www.blogger.com/atom/ns#","term":"minute by minute quotes"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"stock quotes"}],"title":{"type":"text","$t":"Historical Intraday Stock Price Data with Python"},"content":{"type":"html","$t":"By popular request, this post will present how to acquire intraday stock data from google finance using python. The general structure of the code is pretty simple to understand. First a request url is created and sent. The response is then read by python to create an array or matrix of the financial data and a vector of time data. This array is created with the help of the popular Numpy package that can be downloaded from \u003Ca href=\"http:\/\/numpy.scipy.org\/\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. Then in one if-statement, the time data is then restructured into a proper unix time format and translated to a more familiar date string for each financial data point. The translated time vector is then joined with the financial array to produce a single easy to work with financial time series array. Since Numpy has been ported to Python 3, the code I wrote should be compatibile with both Python 2.X and 3.X. Here it is: \u003Csyntaxhighlighter class=\"brush: python;\"\u003Eimport urllib2 import urllib import numpy as np from datetime import datetime urldata = {}  urldata['q'] = ticker = 'JPM'       # stock symbol urldata['x'] = 'NYSE'               # exchange symbol urldata['i'] = '60'                 # interval urldata['p'] = '1d'                 # number of past trading days (max has been 15d) urldata['f'] = 'd,o,h,l,c,v'        # requested data d is time, o is open, c is closing, h is high, l is low, v is volume  url_values = urllib.urlencode(urldata) url = 'http:\/\/www.google.com\/finance\/getprices' full_url = url + '?' + url_values req = urllib2.Request(full_url) response = urllib2.urlopen(req).readlines() getdata = response del getdata[0:7] numberoflines = len(getdata) returnMat = np.zeros((numberoflines, 5)) timeVector = []  index = 0 for line in getdata:     line = line.strip('a')     listFromLine = line.split(',')     returnMat[index,:] = listFromLine[1:6]     timeVector.append(int(listFromLine[0]))     index += 1  # convert Unix or epoch time to something more familiar for x in timeVector:     if x \u0026gt; 500:         z = x         timeVector[timeVector.index(x)] = datetime.fromtimestamp(x)     else:         y = z+x*60 # multiply by interval         timeVector[timeVector.index(x)] = datetime.fromtimestamp(y)  tdata = np.array(timeVector) time = tdata.reshape((len(tdata),1)) intradata = np.concatenate((time, returnMat), axis=1) # array of all data with the properly formated times \u003C\/syntaxhighlighter\u003E\u003Cdiv\u003E\u003Ca name='more'\u003E\u003C\/a\u003EHere is a quick example of what can be done with this data in Python 2.7 using the \u003Ca href=\"http:\/\/numpy.scipy.org\/\" target=\"_blank\"\u003ENumpy\u003C\/a\u003E and \u003Ca href=\"http:\/\/matplotlib.sourceforge.net\/\" target=\"_blank\"\u003Ematplotlib\u003C\/a\u003E libraries. It is nothing fancy, just the usual moving average computations and some styling preferences.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-wxOzBda8Arw\/UCvZS_lvyVI\/AAAAAAAAAXM\/tMMs8YP03JA\/s1600\/jpmquotes.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"480\" src=\"http:\/\/1.bp.blogspot.com\/-wxOzBda8Arw\/UCvZS_lvyVI\/AAAAAAAAAXM\/tMMs8YP03JA\/s640\/jpmquotes.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003EThe python code that was used to create the plot above can is posted below: \u003Csyntaxhighlighter class=\"brush: python;\"\u003E### Example on how to use and plot this data import matplotlib.pyplot as plt import matplotlib.font_manager as font_manager import matplotlib.dates as mdates import matplotlib.ticker as mticker import numpy as np  def relative_strength(prices, n=14):      deltas = np.diff(prices)     seed = deltas[:n+1]     up = seed[seed\u003E=0].sum()\/n     down = -seed[seed\u003C0].sum()\/n     rs = up\/down     rsi = np.zeros_like(prices)     rsi[:n] = 100. - 100.\/(1.+rs)      for i in range(n, len(prices)):         delta = deltas[i-1] # cause the diff is 1 shorter          if delta\u003E0:             upval = delta             downval = 0.         else:             upval = 0.             downval = -delta          up = (up*(n-1) + upval)\/n         down = (down*(n-1) + downval)\/n          rs = up\/down         rsi[i] = 100. - 100.\/(1.+rs)      return rsi  def moving_average(p, n, type='simple'):     \"\"\"     compute an n period moving average.      type is 'simple' | 'exponential'      \"\"\"     p = np.asarray(p)     if type=='simple':         weights = np.ones(n)     else:         weights = np.exp(np.linspace(-1., 0., n))      weights \/= weights.sum()       a =  np.convolve(p, weights, mode='full')[:len(p)]     a[:n] = a[n]     return a  def moving_average_convergence(p, nslow=26, nfast=12):     \"\"\"     compute the MACD (Moving Average Convergence\/Divergence) using a fast and slow exponential moving avg'     return value is emaslow, emafast, macd which are len(p) arrays     \"\"\"     emaslow = moving_average(p, nslow, type='exponential')     emafast = moving_average(p, nfast, type='exponential')     return emaslow, emafast, emafast - emaslow  plt.rc('axes', grid=True) plt.rc('grid', color='0.75', linestyle='-', linewidth=0.5)  textsize = 9 left, width = 0.1, 0.8 rect1 = [left, 0.7, width, 0.2] rect2 = [left, 0.3, width, 0.4] rect3 = [left, 0.1, width, 0.2]   fig = plt.figure(facecolor='white') axescolor  = '#f6f6f6'  # the axies background color  ax1 = fig.add_axes(rect1, axisbg=axescolor)  #left, bottom, width, height ax2 = fig.add_axes(rect2, axisbg=axescolor, sharex=ax1) ax2t = ax2.twinx() ax3  = fig.add_axes(rect3, axisbg=axescolor, sharex=ax1)  ### plot the relative strength indicator prices = intradata[:,4] rsi = relative_strength(prices) t = intradata[:,0] fillcolor = 'darkgoldenrod'  ax1.plot(t, rsi, color=fillcolor) ax1.axhline(70, color=fillcolor) ax1.axhline(30, color=fillcolor) ax1.fill_between(t, rsi, 70, where=(rsi\u003E=70), facecolor=fillcolor, edgecolor=fillcolor) ax1.fill_between(t, rsi, 30, where=(rsi\u003C=30), facecolor=fillcolor, edgecolor=fillcolor) ax1.text(0.6, 0.9, '\u003E70 = overbought', va='top', transform=ax1.transAxes, fontsize=textsize) ax1.text(0.6, 0.1, '\u003C30 = oversold', transform=ax1.transAxes, fontsize=textsize) ax1.set_ylim(0, 100) ax1.set_yticks([30,70]) ax1.text(0.025, 0.95, 'RSI (14)', va='top', transform=ax1.transAxes, fontsize=textsize) ax1.set_title('%s daily'%ticker)  ### plot the price and volume data low = intradata[:,3] high = intradata[:,2]  deltas = np.zeros_like(prices) deltas[1:] = np.diff(prices) up = deltas\u003E0 ax2.vlines(t[up], low[up], high[up], color='black', label='_nolegend_') ax2.vlines(t[~up], low[~up], high[~up], color='black', label='_nolegend_') ma5 = moving_average(prices, 5, type='simple') ma50 = moving_average(prices, 50, type='simple')  linema5, = ax2.plot(t, ma5, color='blue', lw=2, label='MA (5)') linema50, = ax2.plot(t, ma50, color='red', lw=2, label='MA (50)')  props = font_manager.FontProperties(size=10) leg = ax2.legend(loc='center left', shadow=True, fancybox=True, prop=props) leg.get_frame().set_alpha(0.5)  volume = (intradata[:,4]*intradata[:,5])\/1e6  # dollar volume in millions vmax = volume.max() poly = ax2t.fill_between(t, volume, 0, label='Volume', facecolor=fillcolor, edgecolor=fillcolor) ax2t.set_ylim(0, 5*vmax) ax2t.set_yticks([])  ### compute the MACD indicator fillcolor = 'darkslategrey' nslow = 26 nfast = 12 nema = 9 emaslow, emafast, macd = moving_average_convergence(prices, nslow=nslow, nfast=nfast) ema9 = moving_average(macd, nema, type='exponential') ax3.plot(t, macd, color='black', lw=2) ax3.plot(t, ema9, color='blue', lw=1) ax3.fill_between(t, macd-ema9, 0, alpha=0.5, facecolor=fillcolor, edgecolor=fillcolor)   ax3.text(0.025, 0.95, 'MACD (%d, %d, %d)'%(nfast, nslow, nema), va='top',          transform=ax3.transAxes, fontsize=textsize)  # turn off upper axis tick labels, rotate the lower ones, etc for ax in ax1, ax2, ax2t, ax3:     if ax!=ax3:         for label in ax.get_xticklabels():             label.set_visible(False)     else:         for label in ax.get_xticklabels():             label.set_rotation(30)             label.set_horizontalalignment('right')      ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d')  class MyLocator(mticker.MaxNLocator):     def __init__(self, *args, **kwargs):         mticker.MaxNLocator.__init__(self, *args, **kwargs)      def __call__(self, *args, **kwargs):         return mticker.MaxNLocator.__call__(self, *args, **kwargs)  # at most 5 ticks, pruning the upper and lower so they don't overlap # with other ticks ax2.yaxis.set_major_locator(MyLocator(5, prune='both')) ax3.yaxis.set_major_locator(MyLocator(5, prune='both'))  plt.show() \u003C\/syntaxhighlighter\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/2485814839060809259\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/historical-intraday-stock-price-data.html#comment-form","title":"6 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/2485814839060809259"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/2485814839060809259"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/historical-intraday-stock-price-data.html","title":"Historical Intraday Stock Price Data with Python"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-wxOzBda8Arw\/UCvZS_lvyVI\/AAAAAAAAAXM\/tMMs8YP03JA\/s72-c\/jpmquotes.png","height":"72","width":"72"},"thr$total":{"$t":"6"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-874739777315081069"},"published":{"$t":"2012-08-11T20:24:00.000-04:00"},"updated":{"$t":"2012-08-15T13:35:13.974-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"The Walking Dead"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Zombie Apocalypse Modeling"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Zombies"}],"title":{"type":"text","$t":"The Walking Dead Model"},"content":{"type":"html","$t":"Today, I finally discovered a useful application of my chemical engineering degree. To initiate engaging philosophical and mathematical conversations about zombies with grumpy old men.\u003Cbr \/\u003E\u003Cbr \/\u003EAs I sat at my terminal watching the first episode of \u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B005CA4SQK\/ref=as_li_ss_tl?ie=UTF8\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B005CA4SQK\u0026amp;linkCode=as2\u0026amp;tag=nighttimecuri-20\"\u003EThe Walking Dead\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B005CA4SQK\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E, a raspy\u0026nbsp;voice next to me muttered, \"That would never happen.\"\u003Cbr \/\u003E\u003Cbr \/\u003EI replied with the usual, \"Yes, I know zombies don't exist.\" Usually that avoids the unwanted confrontation and allows for a change of subject. It is a formulaic reply that Dale Carnegie would use - if he was a zombie\u0026nbsp;enthusiast. Then he took me by surprise when he explained that the population of a small town would not be completely zombified or dead.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/img.photobucket.com\/albums\/v698\/Leuthen2\/WalkingDead\/WalkingDeadLocations\/TWDLScreencaps\/Hospital001.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"225\" src=\"http:\/\/img.photobucket.com\/albums\/v698\/Leuthen2\/WalkingDead\/WalkingDeadLocations\/TWDLScreencaps\/Hospital001.jpg\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EIn all honesty, I have wondered what I would do in certain zombie apocalypse scenarios. But I have never thought about the larger picture like the population of a town. With a few Auntie Anne's napkins and 45 minutes before boarding we began outlining a population model using the zombie rules from the universe of The Walking Dead.\u003Cbr \/\u003E\u003Ca name='more'\u003E\u003C\/a\u003E\u003Cbr \/\u003EIn The Walking Dead, every person\u0026nbsp;possesses\u0026nbsp;some type of\u0026nbsp;dormant infection that will turn people who die of natural causes into zombies. The bites from these zombies will turn the living into zombies after a period of time. Zombies can be annihilated by the destruction of the reptilian brain. For those readers who are chemical engineers, you should notice that these rules are mathematically very similar to a time dependent batch reaction. The correct model is a simple mass balance, since bodies are conserved.\u003Cbr \/\u003E\u003Cbr \/\u003EThe balance for those who are living includes births, non-zombie related deaths, and those who are bitten from zombie related encounters, respectively:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[\\frac{dL}{dt}=B-nL-xLZ\\]\u003Cbr \/\u003E\u003Cbr \/\u003EThe balance for those who are actively infected includes\u0026nbsp;those who are bitten from zombie related encounters, those who turn into zombies, and the ones that die from non-zombie related causes, respectively:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[\\frac{dI}{dt}=xLZ-pI-nI\\]\u003Cbr \/\u003E\u003Cbr \/\u003EThe balance for zombies contains those who turn into zombies from bites, those who are\u0026nbsp;resurrected\u0026nbsp;from non-zombie related deaths, and those who are killed by an encounter with the living, respectively:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[\\frac{dZ}{dt}=pI-rT-aLZ\\]\u003Cbr \/\u003E\u003Cbr \/\u003EThe final balance is a \"kill\" count that includes the living who died of non-zombie related causes, the actively infected who die of 'natural' causes, zombies that have been destroyed, and the dead who have been resurrected into zombies, respectively:\u003Cbr \/\u003E\u003Cbr \/\u003E\\[\\frac{dT}{dt}=nL+nI+aLZ-rT\\]\u003Cbr \/\u003E\u003Cbr \/\u003EHere is a summary of the notation I used in my model:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cul\u003E\u003Cli\u003EL: the number of living\u003C\/li\u003E\u003Cli\u003EI: the number of bitten\u003C\/li\u003E\u003Cli\u003EZ: the number of zombies\u003C\/li\u003E\u003Cli\u003ET: the number of people \"killed\"\u003C\/li\u003E\u003Cli\u003EB: the population birth rate\u003C\/li\u003E\u003Cli\u003En: the chance of a non-zombie related death\u003C\/li\u003E\u003Cli\u003Ex: the chance of living being bitten in a zombie encounter\u003C\/li\u003E\u003Cli\u003Ep: the chance of a bitten person turning into a zombie\u003C\/li\u003E\u003Cli\u003Er: the chance a dead person is resurrected into a zombie\u003C\/li\u003E\u003Cli\u003Ea: the chance a zombie is totally destroyed\u003C\/li\u003E\u003C\/ul\u003E\u003Cbr \/\u003EWithout access to Mathematica on my laptop, I had to resort to a more old school method, Python. The development of the model and coding went pretty smoothly. It was coming to an agreement on the probabilities of the model that caused us to leave our discussion unresolved before zone 4 boarding. To Tim the\u0026nbsp;taxidermist\u0026nbsp;from Texas and my blog readers, please tell me what you think plausible probabilities are, using the model in the following python code.\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: python;\"\u003E# modeling the walking dead import numpy as np import matplotlib.pyplot as plt from scipy.integrate import odeint  B = 0       # births n = 0.0001  # natural death probability (per day) x = 0.0055  # transmission probability  (per day) r = 0.9000  # resurrect probability (per day) a = 0.0075  # destroy probability  (per day) p = 0.5000  # infection probability  (per day)  # solve the system dy\/dt = f(y, t) def f(y, t):         Li = y[0]         Ii = y[1]         Zi = y[2]         Ti = y[3]         # our modeling equations         f0 = B - x*Li*Zi - n*Li         f1 = x*Li*Zi - p*Ii - n*Ii         f2 = p*Ii + r*Ti - a*Li*Zi         f3 = n*Li + n*Ii + a*Li*Zi - r*Ti         return [f0, f1, f2, f3]  # initial conditions L0 = 500                # initial population I0 = 0                  # initial infected population Z0 = 0                  # initial zombie population T0 = 0                  # initial death population y0 = [L0, I0, Z0, T0]   # initial condition vector t  = np.linspace(0, 30, 5000)   # time grid  # solve the DEs soln = odeint(f, y0, t) L = soln[:, 0] I = soln[:, 1] Z = soln[:, 2] T = soln[:, 3]  # plot results plt.figure() plt.plot(t, L, label='Living') plt.plot(t, Z, label='Zombies') plt.xlabel('Days from outbreak') plt.ylabel('Population') plt.title('Apocalypse through Rules of The Walking Dead') plt.legend(loc=0) #plt.show()  plt.figure() plt.plot(t, L, label='Living') plt.plot(t, I, label='Bitten') plt.xlabel('Days from outbreak') plt.ylabel('Population') plt.title('Apocalypse through Rules of The Walking Dead') plt.legend(loc=0) plt.show() \u003C\/syntaxhighlighter\u003EThe probabilities used in my code were found through some trial and error to closely match those in The Walking Dead series on AMC. Assuming that the zombie outbreak began when Rick Grimes was shot, Rick was in a coma for about three weeks before the military is seen opening fire on the hoards of zombies at the hospital. Then a few days later Rick wakes up to find no living person around. The plots created from the model above replicates this timing pretty well.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-ncnh53qTWRI\/UCb0whGqPRI\/AAAAAAAAAWs\/MN5rk3epcGQ\/s1600\/livingvzombies.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/4.bp.blogspot.com\/-ncnh53qTWRI\/UCb0whGqPRI\/AAAAAAAAAWs\/MN5rk3epcGQ\/s400\/livingvzombies.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-bBDJSgpVkT4\/UCb02C0L-jI\/AAAAAAAAAW0\/2drvCrboTpY\/s1600\/livingvbitten.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/2.bp.blogspot.com\/-bBDJSgpVkT4\/UCb02C0L-jI\/AAAAAAAAAW0\/2drvCrboTpY\/s400\/livingvbitten.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/874739777315081069\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/the-walking-dead-model.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/874739777315081069"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/874739777315081069"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/the-walking-dead-model.html","title":"The Walking Dead Model"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/4.bp.blogspot.com\/-ncnh53qTWRI\/UCb0whGqPRI\/AAAAAAAAAWs\/MN5rk3epcGQ\/s72-c\/livingvzombies.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-788767721217031536"},"published":{"$t":"2012-08-07T11:28:00.002-04:00"},"updated":{"$t":"2012-08-07T11:30:57.790-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"MacPorts"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Mountain Lion"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Scipy"}],"title":{"type":"text","$t":"Installing Scipy from MacPorts on Mountain Lion"},"content":{"type":"html","$t":"After doing a fresh install of Mountain Lion I proceeded with my age old tradition of reinstalling all of my favorite libraries from \u003Ca href=\"http:\/\/www.macports.org\/\" target=\"_blank\"\u003EMacPorts\u003C\/a\u003E. Everything installed wonderfully except for \u003Ca href=\"http:\/\/pyobjc.sourceforge.net\/\" target=\"_blank\"\u003EPyObjC\u003C\/a\u003E, \u003Ca href=\"http:\/\/www.scipy.org\/\" target=\"_blank\"\u003EScipy\u003C\/a\u003E, and a few of their dependencies like gcc45 and libunwind-headers. With an hour of trial and error that included playing with the pip installer and fighting the temptation to switch to \u003Ca href=\"http:\/\/mxcl.github.com\/homebrew\/\" target=\"_blank\"\u003EHomebrew\u003C\/a\u003E, I found an easy and quick fix for MacPorts.\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EIf you have already attempted to install any of those ports I have listed, please clean those ports that have failed to install. For example, my installation of the py27-scipy port failed so I must use the following command to clean it:\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Csyntaxhighlighter class=\"brush: bash\"\u003Esudo port clean --all py27-scipy \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003E\u003Cdiv\u003EThe quick fix to allow MacPorts to install these wonderful ports properly is done by reconfirming the Xcode license agreement with the following command:\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Csyntaxhighlighter class=\"brush: bash\"\u003Esudo xcodebuild -license \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003E\u003Cdiv\u003EContinuing from my example, \u003Ci\u003Esudo port install\u0026nbsp;py27-scipy\u003C\/i\u003E should now execute and install without a hitch. The same goes for the other ports I listed earlier.\u0026nbsp;\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/788767721217031536\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/installing-scipy-from-macports-on.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/788767721217031536"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/788767721217031536"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/08\/installing-scipy-from-macports-on.html","title":"Installing Scipy from MacPorts on Mountain Lion"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-37581094246188300"},"published":{"$t":"2012-07-30T17:10:00.001-04:00"},"updated":{"$t":"2012-07-30T17:10:33.904-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Arduino"},{"scheme":"http://www.blogger.com/atom/ns#","term":"C"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Doctor Who"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Sonic Screwdriver"},{"scheme":"http://www.blogger.com/atom/ns#","term":"TV-B-Gone"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Universal Remote"}],"title":{"type":"text","$t":"Creating a Sonic Screwdriver with an Arduino Nano, Part 1"},"content":{"type":"html","$t":"I was pretty excited to hear about the release of a \u003Ca href=\"http:\/\/www.engadget.com\/2012\/07\/11\/think-geek-mark-vii-sonic-screwdriver-universal-remote\/\" target=\"_blank\"\u003ESonic Screwdriver remote\u003C\/a\u003E. But after watching the YouTube demo, it seems kind of lame. The appeal of the Sonic Screwdriver, used by the good Doctor, comes from its universal applicability to help the Doctor get out of sticky situations or rid general annoyances. A programmable remote just doesn't cut it.\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EFor those in the US who have never caught an old episode of Doctor Who at 5:30 AM in the morning on PBS or watch the newer episodes on \u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B003HYSSZK\" target=\"_blank\"\u003EAmazon Prime\u003C\/a\u003E, a Sonic Screwdriver is akin to\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;MacGyver's swiss army knife, but only more futuristic - the BBC prop makers added an LED to show that it is from the future.\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003Cbr \/\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/www.wired.com\/geekdad\/wp-content\/uploads\/2012\/04\/a28d8ccd461c2af9_289715_327710703909630_127031120644257_1507609_1089709115_o.preview.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"273\" src=\"http:\/\/www.wired.com\/geekdad\/wp-content\/uploads\/2012\/04\/a28d8ccd461c2af9_289715_327710703909630_127031120644257_1507609_1089709115_o.preview.jpg\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003Cbr \/\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003EMy disappointment in the Sonic Screwdriver remote (\u003Ca href=\"http:\/\/www.thinkgeek.com\/product\/ee4a\/\" target=\"_blank\"\u003Eavailable on ThinkGeek\u003C\/a\u003E) has inspired me to design a Sonic Screwdriver that is more true to fiction. I just can't see the Doctor sitting somewhere waving his screwdriver infront of an old Panasonic remote. In this post I will go my plans to integrate a \u003Ca href=\"http:\/\/www.tvbgone.com\/\" target=\"_blank\"\u003ETV-B-Gone\u003C\/a\u003E into a Sonic Screwdriver. In future posts, I may show how to integrate other functions such as\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E128 kHz\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;RFID technology (to open my apartment or work doors) and possibly integrating 13.56 MHz MIFARE\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003Etechnology (in DC we use SmarTrip for public transportation).\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003C\/span\u003E\u003Cbr \/\u003E\u003Ca name='more'\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003EHere is a list of materials necessary for the first screwdriver prototype:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B003M71Z2K\/ref=as_li_ss_tl?ie=UTF8\u0026amp;tag=nighttimecuri-20\u0026amp;linkCode=as2\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B003M71Z2K\" target=\"_blank\"\u003EDoctor Who Eleventh Doctor Die Cast Sonic Screwdriver\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B003M71Z2K\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E \u003Cbr \/\u003E\u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B003YVL34O\/ref=as_li_ss_tl?ie=UTF8\u0026amp;tag=nighttimecuri-20\u0026amp;linkCode=as2\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B003YVL34O\" target=\"_blank\"\u003EArduino Nano v3.0\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B003YVL34O\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E\u003Cbr \/\u003EIR LED (950 nm)\u003Cbr \/\u003E100 Ohm Resistor (I prefer the metal-film ones)\u003Cbr \/\u003ENPN Transistor (I had a 2N4401 laying around)\u003Cbr \/\u003EBreadboard\u003Cbr \/\u003ETact Button\u003Cbr \/\u003ESome Wires\u003Cbr \/\u003EA23 Battery\u003Cbr \/\u003EN-Battery Holder\u003Cbr \/\u003E\u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B000B5YIYS\/ref=as_li_ss_tl?ie=UTF8\u0026amp;tag=nighttimecuri-20\u0026amp;linkCode=as2\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B000B5YIYS\" target=\"_blank\"\u003EWeller WP35 35-Watt Professional Soldering Iron\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B000B5YIYS\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E\u003Cbr \/\u003E\u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B00030AP48\/ref=as_li_ss_tl?ie=UTF8\u0026amp;tag=nighttimecuri-20\u0026amp;linkCode=as2\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B00030AP48\" target=\"_blank\"\u003EAmerican Terminal AT-31604 60-40 Rosin Core Solder (4 Ounces)\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B00030AP48\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003ENow lets get started first by prototyping a TV-B-Gone with the Arduino Nano. Usually this process would take a bit of knowledge of C and microcontroller programming, but luckily \u003Ca href=\"http:\/\/www.arcfn.com\/2010\/11\/improved-arduino-tv-b-gone.html\" target=\"_blank\"\u003EKen Shirriff\u003C\/a\u003E has already created an Arduino port that will serve the purposes of this post. Simply follow the schematics below to set up the Arduino Nano TV-B-Gone that works with Ken Shirriff's \u003Ca href=\"https:\/\/github.com\/shirriff\/Arduino-TV-B-Gone\" target=\"_blank\"\u003Ecode\u003C\/a\u003E.\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-kMUxcVnzdiM\/UBbZo8Z6isI\/AAAAAAAAAVk\/TFSB0xS5UNc\/s1600\/arduino_nano-tvbgone.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"288\" src=\"http:\/\/3.bp.blogspot.com\/-kMUxcVnzdiM\/UBbZo8Z6isI\/AAAAAAAAAVk\/TFSB0xS5UNc\/s400\/arduino_nano-tvbgone.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ETo test the set-up above, connect the Arduino Nano to your computer with a mini-B USB cable. Open up the .pde file downloaded from Ken's blog with the \u003Ca href=\"http:\/\/arduino.cc\/en\/Main\/Software\" target=\"_blank\"\u003EArduino IDE\u003C\/a\u003E. In the tolbar, under Tools \u0026gt; Board, please make sure the correct board is selected. If you ordered the Arduino Nano v3.0 from the link above, the correct selection is listed as \"Arduino Nano w\/ ATmega328.\" Under Tools \u0026gt; Serial Port, make sure the selected port is connected to the Arduino. On Lion OS X 10.7.4, I ran into a slight problem where my mac could not detect the port my Nano was connected on. I was able to fix this by installing an FTDI driver for my system, from \u003Ca href=\"http:\/\/www.ftdichip.com\/Drivers\/VCP.htm\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. Now the correct serial port (\/dev\/tty.usbserial-AE01K9BX) will appear when my Arduino is connected. When both the Board and Serial Port are correctly selected, press the upload button (the right pointing arrow on the top left corner). After uploading is complete, press the tact button on the breadboard and use a digital camera to check that the IR LED is pulsing (pictured below). If it does not pulse, check to see if your connections match the schematic above. You should note that the transistor has it's flat side facing towards you, the short leg on the IR LED is the ground, and the tact button is made up of two separate strips of conductive metal on the bottom. For my European readers, please remember to ground pin 5 or follow \u003Ca href=\"http:\/\/bit.ly\/PgmYKb\" target=\"_blank\"\u003Ethis schematic\u003C\/a\u003E.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-zz9_TKLx0W0\/UBbgVw9R8jI\/AAAAAAAAAV4\/JZb2ggDMKbc\/s1600\/arduino_tvbgone_breadboard.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/3.bp.blogspot.com\/-zz9_TKLx0W0\/UBbgVw9R8jI\/AAAAAAAAAV4\/JZb2ggDMKbc\/s320\/arduino_tvbgone_breadboard.jpg\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003ETo disassemble the head of the Sonic Screwdriver screwdriver, just pull really hard and the insides should come out easily.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-sJe04QFBn-w\/UBbsn-fyBFI\/AAAAAAAAAWU\/eeuG7Ve_VDU\/s1600\/sonicscrewdriver-inside.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"240\" src=\"http:\/\/2.bp.blogspot.com\/-sJe04QFBn-w\/UBbsn-fyBFI\/AAAAAAAAAWU\/eeuG7Ve_VDU\/s320\/sonicscrewdriver-inside.jpg\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EThe next steps will be a bit trickier. I chose to use the Arduino Nano for the board's tiny size, which fits perfectly into the head of the unassembled Sonic Screwdriver. However, the headers of the assembled nano makes it oversized.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-DvrqtWHRNb4\/UBbrOSFSaLI\/AAAAAAAAAWM\/Yh5JNDMYvSY\/s1600\/nano_inside.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"320\" src=\"http:\/\/4.bp.blogspot.com\/-DvrqtWHRNb4\/UBbrOSFSaLI\/AAAAAAAAAWM\/Yh5JNDMYvSY\/s320\/nano_inside.jpg\" width=\"240\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EOne solution is to start the project by buying an unassembled Arduino Nano. The solution I chose was to\u0026nbsp;desolder and remove the headers, which will be covered in my next post. For those of you who just can't wait, all of the necessary parts to complete the project has been presented in this post. To get the basic functions of a TV-B-Gone in a Sonic Screwdriver just solder or \u003Ca href=\"http:\/\/www.amazon.com\/gp\/product\/B000Z9H7ZW\/ref=as_li_ss_tl?ie=UTF8\u0026amp;tag=nighttimecuri-20\u0026amp;linkCode=as2\u0026amp;camp=1789\u0026amp;creative=390957\u0026amp;creativeASIN=B000Z9H7ZW\" target=\"_blank\"\u003Ewire glue\u003C\/a\u003E\u003Cimg alt=\"\" border=\"0\" height=\"1\" src=\"http:\/\/www.assoc-amazon.com\/e\/ir?t=nighttimecuri-20\u0026amp;l=as2\u0026amp;o=1\u0026amp;a=B000Z9H7ZW\" style=\"border: none !important; margin: 0px !important;\" width=\"1\" \/\u003E everything together, connect the N-battery holder to the VIN pin (next to the ground pin), and you are ready to go! The next posts will cover how to assemble the screwdriver with sound effects and increasing the range of the screwdriver."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/37581094246188300\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/creating-sonic-screwdriver-with-arduino.html#comment-form","title":"3 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/37581094246188300"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/37581094246188300"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/creating-sonic-screwdriver-with-arduino.html","title":"Creating a Sonic Screwdriver with an Arduino Nano, Part 1"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/3.bp.blogspot.com\/-kMUxcVnzdiM\/UBbZo8Z6isI\/AAAAAAAAAVk\/TFSB0xS5UNc\/s72-c\/arduino_nano-tvbgone.png","height":"72","width":"72"},"thr$total":{"$t":"3"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-8333916755181043728"},"published":{"$t":"2012-07-27T14:00:00.000-04:00"},"updated":{"$t":"2012-07-28T13:34:12.662-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"C"},{"scheme":"http://www.blogger.com/atom/ns#","term":"eye tracking"},{"scheme":"http://www.blogger.com/atom/ns#","term":"face tracking"},{"scheme":"http://www.blogger.com/atom/ns#","term":"head tracking"},{"scheme":"http://www.blogger.com/atom/ns#","term":"openframeworks"}],"title":{"type":"text","$t":"Face and Eye Tracking Fun!"},"content":{"type":"html","$t":"I recently discovered the \u003Ca href=\"http:\/\/www.openframeworks.cc\/\" target=\"_blank\"\u003Eopenframeworks\u003C\/a\u003E toolkit. It is an open source C++ toolkit that wraps together many commonly used libraries for simple and fast experimentation. I absolutely love it!\u003Cbr \/\u003E\u003Cbr \/\u003EWhen I shared code in the past, it took quite a bit of time to document what libraries, which version, and sometimes (even with C++) which operating system the new user must install for the code to compile properly. With \u003Ca href=\"http:\/\/www.openframeworks.cc\/\" target=\"_blank\"\u003Eopenframeworks\u003C\/a\u003E, that annoyance is a thing of the past.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-rG8l6rfzxp0\/UBLW4zBCYNI\/AAAAAAAAAVQ\/W0G7mtZ29CI\/s1600\/facetracking.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"298\" src=\"http:\/\/1.bp.blogspot.com\/-rG8l6rfzxp0\/UBLW4zBCYNI\/AAAAAAAAAVQ\/W0G7mtZ29CI\/s400\/facetracking.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EFor this post I decided to make a quick and fun example adopted from one of openframeworks' provided examples on detecting faces in pictures. The example can be found under the folder: examples\/addons\/opencvHaarFinderExample.\u003Cbr \/\u003E\u003Cbr \/\u003EI decided to extend their example to detect faces from a live webcam or a quicktime movie. I was amazed at how few lines of code it took. To get started,\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;download the appropriate toolkit for your IDE,\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;\u003C\/span\u003E\u003Ca href=\"http:\/\/www.openframeworks.cc\/download\/\" style=\"background-color: white;\" target=\"_blank\"\u003Ehere\u003C\/a\u003E\u003Cspan style=\"background-color: white;\"\u003E.\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;Then copy the folder\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003EopencvHaarFinderExample in\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003Eexamples\/addons\/ to apps\/myApps\/. Then replace the code in testApp.cpp with the following:\u003C\/span\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: cpp\"\u003E#include \"testApp.h\"  \/\/-------------------------------------------------------------- void testApp::setup(){      camWidth = 480;     camHeight = 360;          \/\/uncomment and comment lines below to test different detectors      finder.setup(\"haarcascade_frontalface_default.xml\");     \/\/finder.setup(\"haarcascade_frontalface_alt.xml\");     \/\/finder.setup(\"haarcascade_frontalface_alt2.xml\");     \/\/finder.setup(\"haarcascade_eyeglasses.xml\");     \/\/finder.setup(\"haarcascade_eye.xml\");   #ifdef _USE_LIVE_VIDEO         VideoGrabber.setVerbose(true);         VideoGrabber.initGrabber(camWidth,camHeight,OF_IMAGE_COLOR);         colorImg.allocate(camWidth,camHeight,OF_IMAGE_COLOR);  #else         VideoPlayer.loadMovie(\"ThemeFromShaft.mov\");         VideoPlayer.play();  #endif  }  \/\/-------------------------------------------------------------- void testApp::update(){   ofBackground(100,100,100);      bool newFrame = false;   #ifdef _USE_LIVE_VIDEO        VideoGrabber.grabFrame();     newFrame = VideoGrabber.isFrameNew();     #else         VideoPlayer.idleMovie();         newFrame = VideoPlayer.isFrameNew();  #endif   if (newFrame){    #ifdef _USE_LIVE_VIDEO            colorImg.setFromPixels(VideoGrabber.getPixels(), camWidth, camHeight, OF_IMAGE_COLOR);         #else            colorImg.setFromPixels(VideoPlayer.getPixels(), camWidth, camHeight, OF_IMAGE_COLOR);         #endif                  finder.findHaarObjects(colorImg);      }  }  \/\/-------------------------------------------------------------- void testApp::draw(){   ofSetHexColor(0xffffff);      colorImg.draw(0,0);   ofNoFill();   for(int i = 0; i \u0026lt; finder.blobs.size(); i++) {            ofRectangle dim = finder.blobs[i].boundingRect;   ofRect(dim.x, dim.y, dim.width, dim.height);           }   ofSetHexColor(0xffffff);  char reportStr[1024];  sprintf(reportStr, \"Number Detected: %lu     fps: %f\", finder.blobs.size(), ofGetFrameRate());  ofDrawBitmapString(reportStr, 20, 20+camHeight);  }   \/\/-------------------------------------------------------------- void testApp::keyPressed  (int key){   switch (key){   case ' ':     break;   } }  \/\/-------------------------------------------------------------- void testApp::mouseMoved(int x, int y ){ }  \/\/-------------------------------------------------------------- void testApp::mouseDragged(int x, int y, int button){ }  \/\/-------------------------------------------------------------- void testApp::mousePressed(int x, int y, int button){ }  \/\/-------------------------------------------------------------- void testApp::mouseReleased(int x, int y, int button){ }  \/\/-------------------------------------------------------------- void testApp::windowResized(int w, int h){ } \u003C\/syntaxhighlighter\u003E\u003Cspan style=\"background-color: white;\"\u003EThen replace the code in testApp.h with the following:\u003C\/span\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: cpp\"\u003E#ifndef _TEST_APP #define _TEST_APP  #include \"ofMain.h\" #include \"ofxCvHaarFinder.h\"  #define _USE_LIVE_VIDEO \/\/comment to use video, uncomment to use webcam  class testApp : public ofBaseApp{   public:    void setup();   void update();   void draw();    void keyPressed  (int key);   void mouseMoved(int x, int y );   void mouseDragged(int x, int y, int button);   void mousePressed(int x, int y, int button);   void mouseReleased(int x, int y, int button);   void windowResized(int w, int h);          int camWidth, camHeight;          #ifdef _USE_LIVE_VIDEO     ofVideoGrabber   VideoGrabber;   #else     ofVideoPlayer   VideoPlayer;   #endif          ofxCvHaarFinder         finder;          ofImage                 colorImg;   };  #endif  \u003C\/syntaxhighlighter\u003EFor those mac users out there, just download \u003Ca href=\"http:\/\/bit.ly\/MKnbuF\"\u003Ethis\u003C\/a\u003E, open \/apps\/myApps\/HeadTracking\/TrackingExample.xcodeproj in Xcode (3.2 and above) and hit run!"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/8333916755181043728\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/face-and-eye-tracking-fun.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8333916755181043728"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/8333916755181043728"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/face-and-eye-tracking-fun.html","title":"Face and Eye Tracking Fun!"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-rG8l6rfzxp0\/UBLW4zBCYNI\/AAAAAAAAAVQ\/W0G7mtZ29CI\/s72-c\/facetracking.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-1692118792063164201"},"published":{"$t":"2012-07-11T13:24:00.002-04:00"},"updated":{"$t":"2012-07-11T13:24:55.375-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"backfill stock quotes"},{"scheme":"http://www.blogger.com/atom/ns#","term":"google finance"},{"scheme":"http://www.blogger.com/atom/ns#","term":"MATLAB"},{"scheme":"http://www.blogger.com/atom/ns#","term":"minute by minute quotes"},{"scheme":"http://www.blogger.com/atom/ns#","term":"stock quotes"}],"title":{"type":"text","$t":"Historical Minute-by-Minute Stock Prices in MATLAB"},"content":{"type":"html","$t":"This is an extension of \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/04\/google-finance-backfill.html\" target=\"_blank\"\u003Emy first post\u003C\/a\u003E on this blog. My first post went over how to gather the minute to minute stock price data from Google Finance using Mathematica. Using the same general programming structure, I've created a function to do this in MATLAB.\u0026nbsp;\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003Cbr \/\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv\u003EThe IntraDayStockData function can be obtained \u003Ca href=\"http:\/\/bit.ly\/Lfvm0k\"\u003Ehere\u003C\/a\u003E.\u0026nbsp;\u003Cspan style=\"background-color: white;\"\u003EThis function takes necessary inputs of a stock symbol and the name of the exchange the stock is traded in. This function also takes optional inputs of quote\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003Efrequency\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;(in seconds) and number of previous trading days. By default, the optional inputs are set for 1 minute quote intervals and 15 previous trading days. The output of this function are the time, the time in string format, trade volume, highest prices, lowest prices, and closing prices. Here is a quick example for getting the minute by minute quotes of the latest trading day (today) for JP Morgan Chase:\u003C\/span\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: matlabkey\"\u003Ejpm = IntraDayStockData('JPM','NYSE','60','1d'); plot(jpm.date,jpm.close,'b-'); hold on; plot(jpm.date,jpm.high,'r-'); hold on; plot(jpm.date,jpm.low,'g-'); datetick('x',16); \u003C\/syntaxhighlighter\u003E\u003Cdiv\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003Cbr \/\u003E\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-LwczQDKK8D4\/T_2zmIAIg0I\/AAAAAAAAAU8\/kDyGzNgrHLA\/s1600\/jpmexample.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"300\" src=\"http:\/\/2.bp.blogspot.com\/-LwczQDKK8D4\/T_2zmIAIg0I\/AAAAAAAAAU8\/kDyGzNgrHLA\/s400\/jpmexample.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/1692118792063164201\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/historical-minute-by-minute-stock.html#comment-form","title":"4 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1692118792063164201"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1692118792063164201"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/historical-minute-by-minute-stock.html","title":"Historical Minute-by-Minute Stock Prices in MATLAB"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-LwczQDKK8D4\/T_2zmIAIg0I\/AAAAAAAAAU8\/kDyGzNgrHLA\/s72-c\/jpmexample.png","height":"72","width":"72"},"thr$total":{"$t":"4"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-891051457986529809"},"published":{"$t":"2012-07-03T12:38:00.000-04:00"},"updated":{"$t":"2012-07-03T12:38:39.007-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Neighborhood Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"GIS"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Homicides"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"}],"title":{"type":"text","$t":"Where are the Homicides in Chicago?"},"content":{"type":"html","$t":"Recently, CNN did a \u003Ca href=\"http:\/\/www.cnn.com\/video\/#\/video\/bestoftv\/2012\/07\/02\/exp-eb-chicago-violence-crisis.cnn?iref=allsearch\" target=\"_blank\"\u003Ereport\u003C\/a\u003E on the high number of homicides that is plaguing Chicago. One of the sure fire solutions suggested was to place more police in the areas where these killings occur. It got me wondering where these homicides actually occur.\u003Cbr \/\u003E\u003Cbr \/\u003EUsing the code from my \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2012\/05\/chicago-crime-updated.html\" target=\"_blank\"\u003Eprevious post\u003C\/a\u003E, I called the crime API that the city of Chicago provides and plotted out all the reported homicides between August 1st, 2011 and May 31st, 2012. There were 403 in total.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-efUFCs9pkKQ\/T_MbfzBbtkI\/AAAAAAAAAUY\/8aJBOKV1ivM\/s1600\/Chicago_Homicides.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"190\" src=\"http:\/\/1.bp.blogspot.com\/-efUFCs9pkKQ\/T_MbfzBbtkI\/AAAAAAAAAUY\/8aJBOKV1ivM\/s640\/Chicago_Homicides.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: white;\"\u003ENow where did these incidents occur?\u003C\/span\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-S8WTeKym-94\/T_MbmuIvTwI\/AAAAAAAAAUg\/MU1NlX8SzzI\/s1600\/Chicago_HomicidesGIS.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"640\" src=\"http:\/\/1.bp.blogspot.com\/-S8WTeKym-94\/T_MbmuIvTwI\/AAAAAAAAAUg\/MU1NlX8SzzI\/s640\/Chicago_HomicidesGIS.png\" width=\"534\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EInitially it looks as if the homicides are wide spread throughout the city. However, if we do a breakdown by neighborhood it becomes obvious that some are much worse off than others.\u0026nbsp;\u003Cspan style=\"background-color: white;\"\u003EBelow is a table of each Chicago neighborhood and its associated number of homicides\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003Ebetween August 1st, 2011 and May 31st, 2012\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E. I've also provided a more\u0026nbsp;convenient\u0026nbsp;graphical representation of this table. The Mathematica code and data files I used for this post can be found \u003Ca href=\"http:\/\/bit.ly\/M3ICBm\" target=\"_blank\"\u003Ehere\u003C\/a\u003E.\u003C\/span\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"post-column-left\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-KNwWnINnjqA\/T_MbucjGT4I\/AAAAAAAAAUo\/R0Yz5iSxP3A\/s1600\/Chicago_HomicidesbyNeighborhoods.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-KNwWnINnjqA\/T_MbucjGT4I\/AAAAAAAAAUo\/R0Yz5iSxP3A\/s1600\/Chicago_HomicidesbyNeighborhoods.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cdiv class=\"post-column-right\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-mYM2SwXF7Ns\/T_Mb4N9c7PI\/AAAAAAAAAUw\/_1YtUvTaXQI\/s1600\/Chicago_HomicideDensity.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"320\" src=\"http:\/\/2.bp.blogspot.com\/-mYM2SwXF7Ns\/T_Mb4N9c7PI\/AAAAAAAAAUw\/_1YtUvTaXQI\/s320\/Chicago_HomicideDensity.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/891051457986529809\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/where-are-homicides-in-chicago.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/891051457986529809"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/891051457986529809"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/07\/where-are-homicides-in-chicago.html","title":"Where are the Homicides in Chicago?"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-efUFCs9pkKQ\/T_MbfzBbtkI\/AAAAAAAAAUY\/8aJBOKV1ivM\/s72-c\/Chicago_Homicides.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-3112370049733525166"},"published":{"$t":"2012-06-27T16:20:00.001-04:00"},"updated":{"$t":"2012-06-28T13:24:13.139-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Crime Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Hidden Markov Model"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"}],"title":{"type":"text","$t":"Hidden Markov Model of Chicago Crimes"},"content":{"type":"html","$t":"This is a bit of a side note from my attempts to model the number of daily crimes in the city of Chicago. I've been treating the Chicago crime data as a parametric random process where the parameters of the stochastic process can be estimated in a well-defined manner. My first few attempts used various autoregressive models that assumed the crime data was a signal that cross-correlated with itself. From my knowledge of statistics, this is the proper and prescribed treatment for the data. But I have been wondering how well a strictly stochastic model like the Hidden Markov Model (HMM) will perform.\u003Cbr \/\u003E\u003Cbr \/\u003EA hidden Markov model is not that hard of a conceptual stretch when applied to crime modeling. Lets assume that all criminal activities in Chicago evolved according to a Markov process, which may have a finite or infinite amount of states. The underlying Markov process is not directly observable because these states are hidden. Luckily, for each state there is an associated stochastic process which produces an observable signal (a crime report). So as this criminal system in Chicago evolves over time, delegated by the hidden Markov chain, it produces crime reports as observable outcomes.\u003Cbr \/\u003E\u003Cbr \/\u003EFor a full\u0026nbsp;explanation\u0026nbsp;of the mathematical theory behind HMMs please refer to \u003Ca href=\"http:\/\/www.cs.cornell.edu\/Courses\/cs4758\/2012sp\/materials\/hmm_paper_rabiner.pdf\" target=\"_blank\"\u003Ethis\u003C\/a\u003E article by Lawrence Rabiner. Even though Rabiner focuses on HMM applications to speech recognition, it covers in great detail the theory used in my implementation. To simplify things for myself and to cut down on computation time, I created a hidden Markov model with univariate Gaussian outcomes. The model is fitted using a maximum likelihood estimation, where the values of the parameters are those that maximizes the likelihood of the observed data. Here, I used the \u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Baum%E2%80%93Welch_algorithm\" target=\"_blank\"\u003EBaum-Welch Algorithm\u003C\/a\u003E for estimating the maximum likelihood.\u003Cbr \/\u003E\u003Cbr \/\u003EThe observed Chicago crime data is the total number of daily reported crimes between July 1, 2011 and March 31, 2012. A log difference transformation of this data was performed to create a more normal distribution of observables. This is all done in Mathematica with six lines of code: \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003Ecsv = Import[\u003Cbr \/\u003E   \"https:\/\/data.cityofchicago.org\/api\/views\/x2n5-8w5q\/rows.csv\", \u003Cbr \/\u003E   \"CSV\"];\u003Cbr \/\u003E\u003Cbr \/\u003Estart = \"07\/01\/2011\";  end = \"03\/31\/2012\";\u003Cbr \/\u003E\u003Cbr \/\u003Erange = Join[\u003Cbr \/\u003E   First[Position[\u003Cbr \/\u003E      If[StringMatchQ[#, start ~~ __], 1, 0] \u0026amp; \/@ \u003Cbr \/\u003E       Drop[csv, 1][[All, 2]], 1]] + 1, \u003Cbr \/\u003E   Last[Position[\u003Cbr \/\u003E      If[StringMatchQ[#, end ~~ __], 1, 0] \u0026amp; \/@ \u003Cbr \/\u003E       Drop[csv, 1][[All, 2]], 1]] + 1];\u003Cbr \/\u003E\u003Cbr \/\u003Ecrimeseries = \u003Cbr \/\u003E  Tally[DateString[\u003Cbr \/\u003E      DateList[{ToString[#], {\"Month\", \"\/\", \"Day\", \"\/\", \"Year\", \" \", \u003Cbr \/\u003E         \"Hour12\", \":\", \"Minute\", \" \", \"AMPM\"}}], {\"MonthName\", \" \", \u003Cbr \/\u003E       \"Day\", \" \", \"Year\"}] \u0026amp; \/@ Take[csv, range][[All, 2]]];\u003Cbr \/\u003E\u003C\/pre\u003EThe HMM fitting of the crime data uses three functions I created in Mathematica: \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003EMarkovChainEquilibrium[TransitionMatrix_] := \u003Cbr \/\u003E  Inverse[Transpose[TransitionMatrix] - \u003Cbr \/\u003E     IdentityMatrix[Length[TransitionMatrix]] + 1].ConstantArray[1, \u003Cbr \/\u003E    Length[TransitionMatrix]];\u003Cbr \/\u003E\u003Cbr \/\u003ERandomInitial[Data_, States_] := Module[\u003Cbr \/\u003E   {TransMat, i, vnGamma, ProbVec, StateMean, vnQ, StateStD, t, mnW},\u003Cbr \/\u003E   TransMat = (#\/Total[#]) \u0026amp; \/@ (RandomReal[{0.01, 0.99}, {States, \u003Cbr \/\u003E        States}]);\u003Cbr \/\u003E   ProbVec = MarkovChainEquilibrium[TransMat];\u003Cbr \/\u003E   StateMean = ((#\/Total[#]) \u0026amp;[\u003Cbr \/\u003E       RandomReal[{-0.99, 0.99}, States]]) Mean[Data];\u003Cbr \/\u003E   StateStD = \u003Cbr \/\u003E    Sqrt[((#\/Total[#]) \u0026amp;[RandomReal[{0.1, 0.99}, States]]) Variance[\u003Cbr \/\u003E       Data]];\u003Cbr \/\u003E   {ProbVec, TransMat, StateMean, StateStD}];\u003Cbr \/\u003E\u003Cbr \/\u003EOptions[HMM] = {\"LikelihoodTolerance\" -\u0026gt; 10^(-7), \u003Cbr \/\u003E   \"MaxIterations\" -\u0026gt; 500};\u003Cbr \/\u003E\u003Cbr \/\u003EHMM[Data_, {InitialIota_, InitialA_, InitialMean_, InitialStD_}, \u003Cbr \/\u003E   OptionsPattern[]] := Module[\u003Cbr \/\u003E   {TransMat, matAlpha, matB, matBeta, nBIC, matGamma, ProbVec, \u003Cbr \/\u003E    StateLogLikelihood, iMaxIter, StateMean, iN, StateNu, StateStD, t,\u003Cbr \/\u003E     iT, nTol, matWeights, matXi},\u003Cbr \/\u003E   nTol = OptionValue[\"LikelihoodTolerance\"]; \u003Cbr \/\u003E   iMaxIter = OptionValue[\"MaxIterations\"];\u003Cbr \/\u003E   iT = Length[Data]; iN = Length[InitialA];\u003Cbr \/\u003E   ProbVec = InitialIota; TransMat = InitialA; \u003Cbr \/\u003E   StateMean = InitialMean; StateStD = InitialStD;\u003Cbr \/\u003E   matB = \u003Cbr \/\u003E    Table[PDF[NormalDistribution[StateMean[[#]], StateStD[[#]]], \u003Cbr \/\u003E        Data[[t]]] \u0026amp; \/@ Range[iN], {t, 1, iT}];\u003Cbr \/\u003E   matAlpha = Array[0. \u0026amp;, {iT, iN}];\u003Cbr \/\u003E   StateNu = Array[0. \u0026amp;, iT];\u003Cbr \/\u003E   matAlpha[[1]] = ProbVec matB[[1]];\u003Cbr \/\u003E   StateNu[[1]] = 1\/Total[matAlpha[[1]]];\u003Cbr \/\u003E   matAlpha[[1]] *= StateNu[[1]];\u003Cbr \/\u003E   For[t = 2, t \u0026lt;= iT, t++, \u003Cbr \/\u003E    matAlpha[[t]] = (matAlpha[[t - 1]].TransMat) matB[[t]]; \u003Cbr \/\u003E    StateNu[[t]] = 1\/Total[matAlpha[[t]]]; \u003Cbr \/\u003E    matAlpha[[t]] *= StateNu[[t]]];\u003Cbr \/\u003E   StateLogLikelihood = {-Total[Log[StateNu]]};\u003Cbr \/\u003E   Do[matBeta = Array[0. \u0026amp;, {iT, iN}]; \u003Cbr \/\u003E    matBeta[[iT, ;;]] = StateNu[[iT]];\u003Cbr \/\u003E    For[t = iT - 1, t \u0026gt;= 1, t--, \u003Cbr \/\u003E     matBeta[[t]] = \u003Cbr \/\u003E      TransMat.(matBeta[[t + 1]] matB[[t + 1]]) StateNu[[t]]];\u003Cbr \/\u003E    matGamma = (#\/Total[#]) \u0026amp; \/@ (matAlpha matBeta);\u003Cbr \/\u003E    matXi = Array[0. \u0026amp;, {iN, iN}];\u003Cbr \/\u003E    For[t = 1, t \u0026lt;= iT - 1, t++, \u003Cbr \/\u003E     matXi += (#\/Total[Flatten[#]]) \u0026amp;[\u003Cbr \/\u003E      TransMat KroneckerProduct[matAlpha[[t]], \u003Cbr \/\u003E        matBeta[[t + 1]] matB[[t + 1]]]]];\u003Cbr \/\u003E    TransMat = (#\/Total[#]) \u0026amp; \/@ matXi;\u003Cbr \/\u003E    ProbVec = matGamma[[1]];\u003Cbr \/\u003E    matWeights = (#\/Total[#]) \u0026amp; \/@ Transpose[matGamma];\u003Cbr \/\u003E    StateMean = matWeights.Data;\u003Cbr \/\u003E    StateStD = Sqrt[Total \/@ (matWeights (Data - # \u0026amp; \/@ StateMean)^2)];\u003Cbr \/\u003E    matB = \u003Cbr \/\u003E     Table[PDF[NormalDistribution[StateMean[[#]], StateStD[[#]]], \u003Cbr \/\u003E         Data[[t]]] \u0026amp; \/@ Range[iN], {t, 1, iT}];\u003Cbr \/\u003E    matAlpha = Array[0. \u0026amp;, {iT, iN}];\u003Cbr \/\u003E    StateNu = Array[0. \u0026amp;, iT];\u003Cbr \/\u003E    matAlpha[[1]] = ProbVec matB[[1]];\u003Cbr \/\u003E    StateNu[[1]] = 1\/Total[matAlpha[[1]]];\u003Cbr \/\u003E    matAlpha[[1]] *= StateNu[[1]];\u003Cbr \/\u003E    For[t = 2, t \u0026lt;= iT, t++, \u003Cbr \/\u003E     matAlpha[[t]] = (matAlpha[[t - 1]].TransMat) matB[[t]]; \u003Cbr \/\u003E     StateNu[[t]] = 1\/Total[matAlpha[[t]]]; \u003Cbr \/\u003E     matAlpha[[t]] *= StateNu[[t]]];\u003Cbr \/\u003E    StateLogLikelihood = \u003Cbr \/\u003E     Append[StateLogLikelihood, -Total[Log[StateNu]]];\u003Cbr \/\u003E    If[StateLogLikelihood[[-1]]\/StateLogLikelihood[[-2]] - 1 \u0026lt;= nTol, \u003Cbr \/\u003E     Break[]],\u003Cbr \/\u003E    {iMaxIter}];\u003Cbr \/\u003E   nBIC = -2 StateLogLikelihood[[-1]] + (iN (iN + 2) - 1) Log[iT];\u003Cbr \/\u003E   {\"\\[Iota]\" -\u0026gt; ProbVec, \"a\" -\u0026gt; TransMat, \"\\[Mu]\" -\u0026gt; StateMean, \u003Cbr \/\u003E    \"\\[Sigma]\" -\u0026gt; StateStD, \"\\[Gamma]\" -\u0026gt; matGamma, \"BIC\" -\u0026gt; nBIC, \u003Cbr \/\u003E    \"LL\" -\u0026gt; StateLogLikelihood}];\u003Cbr \/\u003E\u003C\/pre\u003EThe first function computes the\u0026nbsp;equilibrium\u0026nbsp;distribution for an ergodic Markov chain. The second function generates a random parameter set with inputs of the observed data and the number of states. Output from the second function is a list containing the initial state probability vector, the transition matrix of the Markov chain, the mean of each state, and the standard deviation of each state. The third function is the maximum likelihood estimated fit of the HMM. It takes inputs of the observed data and a random parameter set from the second function. The output from the third function is a list that contains the initial state probability vector, the transition matrix, the mean vector, the standard deviation vector, a vector of periodic state probabilities, the Bayesian Information Criterion (BIC) for the fit, and a history of the loglikelihood for each iteration.\u003Cbr \/\u003E\u003Cbr \/\u003EAs an example of how to use these functions, lets generate a two state fit with the log differenced crime data. The first step is to generate a random initial parameter set. To hedge the risk of generating a few really bad initial guesses I chose to generate a set of 20 initial guess and run them for 10 iterations. From this set, I then chose the candidate with the highest loglikelihood. This candidate will then be used as a starting point for fitting the model. \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003Eguess = HMM[crimedifference[[All, 2]], #, \"MaxIterations\" -\u0026gt; 10] \u0026amp; \/@ \u003Cbr \/\u003E   Table[RandomInitial[crimedifference[[All, 2]], 2], {20}];\u003Cbr \/\u003E\u003Cbr \/\u003EbestguessLL = \u003Cbr \/\u003E  Flatten[Position[#, Max[#]] \u0026amp;[Last[\"LL\" \/. #] \u0026amp; \/@ guess]][[1]];\u003Cbr \/\u003E\u003Cbr \/\u003Ehmmfit = HMM[\u003Cbr \/\u003E   crimedifference[[All, \u003Cbr \/\u003E    2]], {\"\\[Iota]\", \"a\", \"\\[Mu]\", \u003Cbr \/\u003E     \"\\[Sigma]\"} \/. (guess[[bestguessLL]])];\u003Cbr \/\u003E\u003C\/pre\u003EIt is usually a good idea to check the convergence of the likelihood function to make sure nothing has gone awry. \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003EListPlot[\"LL\" \/. hmmfit, PlotRange -\u0026gt; All]\u003Cbr \/\u003E\u003C\/pre\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-pAb73HKcw98\/T-tnbmjCJAI\/AAAAAAAAAT0\/QZrG2eXjfXU\/s1600\/convergence.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"184\" src=\"http:\/\/4.bp.blogspot.com\/-pAb73HKcw98\/T-tnbmjCJAI\/AAAAAAAAAT0\/QZrG2eXjfXU\/s320\/convergence.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ETo cut down on unnecessary typing, I have written a Do loop to evaluate models with different number of states and used a For loop to create a table of the different models with their associated BIC. \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003EMaxGaussians = 3;\u003Cbr \/\u003E\u003Cbr \/\u003EDo[InitialGuess[\"Crimes\", i] = \u003Cbr \/\u003E  HMM[crimedifference[[All, 2]], #, \"MaxIterations\" -\u0026gt; 10] \u0026amp; \/@ \u003Cbr \/\u003E   Table[RandomInitial[crimedifference[[All, 2]], i], {20}];\u003Cbr \/\u003E BestGuessLL[\"Crimes\", i] = \u003Cbr \/\u003E  Flatten[Position[#, Max[#]] \u0026amp;[\u003Cbr \/\u003E     Last[\"LL\" \/. #] \u0026amp; \/@ InitialGuess[\"Crimes\", i]]][[1]];\u003Cbr \/\u003E FittedHMM[\"Crimes\", i] = \u003Cbr \/\u003E  HMM[crimedifference[[All, \u003Cbr \/\u003E    2]], {\"\\[Iota]\", \"a\", \"\\[Mu]\", \u003Cbr \/\u003E     \"\\[Sigma]\"} \/. (InitialGuess[\"Crimes\", i][[\u003Cbr \/\u003E      BestGuessLL[\"Crimes\", i]]])],\u003Cbr \/\u003E {i, 1, MaxGaussians, 1}]\u003Cbr \/\u003E\u003Cbr \/\u003Ebic = {}; states = {};\u003Cbr \/\u003E\u003Cbr \/\u003EFor[i = 1, i \u0026lt; MaxGaussians + 1, i++, AppendTo[states, i]; \u003Cbr \/\u003E j = \"BIC\" \/. FittedHMM[\"Crimes\", i]; AppendTo[bic, j]]\u003Cbr \/\u003E\u003Cbr \/\u003ETableForm[Transpose[{states, bic}], \u003Cbr \/\u003E TableHeadings -\u0026gt; {None, {\"#States\", \"BIC\"}}, \u003Cbr \/\u003E TableAlignments -\u0026gt; Center]\u003Cbr \/\u003E\u003C\/pre\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-RKfwINWonZA\/T-tnp5pwMXI\/AAAAAAAAAT8\/bIgavWbS3-U\/s1600\/bictable.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-RKfwINWonZA\/T-tnp5pwMXI\/AAAAAAAAAT8\/bIgavWbS3-U\/s1600\/bictable.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EBy the BIC, it seems that a HMM with two univariate Gaussian states is the best model. If we examine the Markov chain equilibrium\u0026nbsp;distribution, it seems crimes in Chicago are at a highly volatile with decreasing crimes state about 91% of the time and a low volatility with moderate increases in crimes state about 9% of the time. Looking at the main diagonal of the transition matrix, the large transitions indicate that if crimes in Chicago are found in one state then it will stay in that state for a while. \u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003EPrint[\"Equilibrium Distribution = \", \u003Cbr \/\u003E MatrixForm[MarkovChainEquilibrium[\"a\" \/. FittedHMM[\"Crimes\", 2]]]]\u003Cbr \/\u003E\u003Cbr \/\u003EPrint[\"Transition Matrix from Data = \", \u003Cbr \/\u003E MatrixForm[\"a\" \/. FittedHMM[\"Crimes\", 2]]]\u003Cbr \/\u003E\u003C\/pre\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-qGAEhBEq9Yk\/T-tn5fCe2YI\/AAAAAAAAAUE\/vrIBeuIDawU\/s1600\/eqdist.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-qGAEhBEq9Yk\/T-tn5fCe2YI\/AAAAAAAAAUE\/vrIBeuIDawU\/s1600\/eqdist.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-0LN-o8Jp8bo\/T-tn-6FpEaI\/AAAAAAAAAUM\/VZYzCacQbHQ\/s1600\/transmat.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"26\" src=\"http:\/\/3.bp.blogspot.com\/-0LN-o8Jp8bo\/T-tn-6FpEaI\/AAAAAAAAAUM\/VZYzCacQbHQ\/s320\/transmat.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThat is all for now, I'll try to create a simulation with the HMM data for the next post. All the files used for this post can be found \u003Ca href=\"http:\/\/bit.ly\/LtELfJ\" target=\"_blank\"\u003Ehere\u003C\/a\u003E."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/3112370049733525166\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/hidden-markov-model-of-chicago-crimes.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3112370049733525166"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3112370049733525166"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/hidden-markov-model-of-chicago-crimes.html","title":"Hidden Markov Model of Chicago Crimes"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/4.bp.blogspot.com\/-pAb73HKcw98\/T-tnbmjCJAI\/AAAAAAAAAT0\/QZrG2eXjfXU\/s72-c\/convergence.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-3355288266509650400"},"published":{"$t":"2012-06-26T10:20:00.001-04:00"},"updated":{"$t":"2012-06-28T13:25:08.203-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Autoregressive modeling"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Crime Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"}],"title":{"type":"text","$t":"First Attempt at Modeling Crime in Chicago, Part 3"},"content":{"type":"html","$t":"I've been trying to keep my attempts to model crimes in Chicago relatively simple. I was hoping to be able to generate some decent predictions by examining a few months of crime data as a signal using a simple autoregressive moving average model (ARMA). And I've finally made some progress!\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-rGEtvdZnLW0\/T-ieuwiPl_I\/AAAAAAAAASg\/Xb4mLksfm1E\/s1600\/Chicago_Crime-ARMASimulation1.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"208\" src=\"http:\/\/2.bp.blogspot.com\/-rGEtvdZnLW0\/T-ieuwiPl_I\/AAAAAAAAASg\/Xb4mLksfm1E\/s640\/Chicago_Crime-ARMASimulation1.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-4iqjj9haMHo\/T-ie0TdtZmI\/AAAAAAAAASo\/2oxPwR_RXhc\/s1600\/Chicago_Crime-ARMASimulation2.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"208\" src=\"http:\/\/3.bp.blogspot.com\/-4iqjj9haMHo\/T-ie0TdtZmI\/AAAAAAAAASo\/2oxPwR_RXhc\/s640\/Chicago_Crime-ARMASimulation2.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EI found that a ARMA(2, 3) process acts as a reasonable descriptor for the first difference of the crime data between July 1st, 2011 and March 31st, 2012. The graphs above are simulations for April 2012 that were generated with my regressed ARMA(2, 3) parameters. My choices of using an ARMA model and fitting the first difference of the data is explained in a \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in_12.html\" target=\"_blank\"\u003Eprevious post\u003C\/a\u003E. The Mathematica code that performs the fitting and diagnostic checks is quite extensive so I may release the work for this post as a package in a later post and document all of the functions then. But if you want to play around with my code and do not mind working with Mathematica, you can find everything used to create this post \u003Ca href=\"http:\/\/bit.ly\/KKofwy\" target=\"_blank\"\u003Ehere\u003C\/a\u003E (Warning! It is still a work in progress)."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/3355288266509650400\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in_26.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3355288266509650400"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3355288266509650400"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in_26.html","title":"First Attempt at Modeling Crime in Chicago, Part 3"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-rGEtvdZnLW0\/T-ieuwiPl_I\/AAAAAAAAASg\/Xb4mLksfm1E\/s72-c\/Chicago_Crime-ARMASimulation1.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-5129865044444423769"},"published":{"$t":"2012-06-21T13:25:00.001-04:00"},"updated":{"$t":"2012-06-28T13:26:08.024-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"blogger"},{"scheme":"http://www.blogger.com/atom/ns#","term":"google-code-prettify"},{"scheme":"http://www.blogger.com/atom/ns#","term":"HTML"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"SyntaxHighlighter"}],"title":{"type":"text","$t":"Simple Step to Create a Slicker Site"},"content":{"type":"html","$t":"In my last few posts I had been skimping on a great deal of procedural details and presented only on some fun results. This could be blamed on my dwindling resources of free time and my compulsion to make things presentable. These reasons are\u0026nbsp;significant\u0026nbsp;hurdles to blogging about a semi-programming language like Mathematica, a document markup language like LaTeX, or intermediate-level languages like C++. Other than displaying these codes as plaintext or a cropped screenshot, there are few efficient options to posting them and even fewer to display the code's syntax in a manner that would fit a personalized blog design on Blogger.\u003Cbr \/\u003E\u003Cbr \/\u003EA fantastic solution to this problem is the combined use of\u0026nbsp;\u003Ca href=\"http:\/\/code.google.com\/p\/google-code-prettify\/\" target=\"_blank\"\u003Egoogle-code-prettify\u003C\/a\u003E\u0026nbsp;and\u0026nbsp;\u003Ca href=\"http:\/\/alexgorbatchev.com\/SyntaxHighlighter\/\" target=\"_blank\"\u003ESyntaxHighlighter\u003C\/a\u003E. The setup of these JavaScripts on Blogger is quite simple, the\u0026nbsp;configurations\u0026nbsp;are limitless, and the results are visually appealing.\u003Cbr \/\u003E\u003Cbr \/\u003EFor example, if I wanted to write a quick crazy mathematica post on how to gather all the historical drawings for the \u003Ca href=\"http:\/\/www.megamillions.com\/\" target=\"_blank\"\u003EMega Millions\u003C\/a\u003E lottery. It would've taken me some time to create a Blogger-ready-sized png picture of the code to upload to Blogger and it would've annoyed my blog readers because they would have to manually type out the code below.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-LpNteplfdSc\/T-NEd3DHTuI\/AAAAAAAAASQ\/aBL5Y46t9OE\/s1600\/MegaMillions.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"104\" src=\"http:\/\/2.bp.blogspot.com\/-LpNteplfdSc\/T-NEd3DHTuI\/AAAAAAAAASQ\/aBL5Y46t9OE\/s640\/MegaMillions.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EUsing google-code-prettify I can copy and paste my code in nanoseconds and my readers would also be able to use the code to pick their winning Mega Millions numbers right away.\u003Cbr \/\u003E\u003Cpre class=\"prettyprint lang-mma\"\u003Edat[x_] := \u003Cbr \/\u003E Drop[Drop[\u003Cbr \/\u003E   Flatten[Import[\u003Cbr \/\u003E     \"http:\/\/www.usamega.com\/mega-millions-history.asp?p=\" \u0026lt;\u0026gt; \u003Cbr \/\u003E      ToString[x] \u0026lt;\u0026gt; \"\", \"Data\"], 4], 5], -3]\u003Cbr \/\u003En = 61;\u003Cbr \/\u003Edat1 = ToString[Flatten[dat[#] \u0026amp; \/@ Range[n], 1]];\u003Cbr \/\u003Edata = ToExpression[\u003Cbr \/\u003E   StringReplace[dat1, {\"\\[CenterDot]\" -\u0026gt; \",\", \"+\" -\u0026gt; \",\"}]];\u003Cbr \/\u003Enumber1 = data[[All, 4]];\u003Cbr \/\u003Enumber2 = data[[All, 5]];\u003Cbr \/\u003Enumber3 = data[[All, 6]];\u003Cbr \/\u003Enumber4 = data[[All, 7]];\u003Cbr \/\u003Enumber5 = data[[All, 8]];\u003Cbr \/\u003Emegaball = data[[All, 9]];\u003Cbr \/\u003E\u003C\/pre\u003ETo get google-code-prettify set up on Blogger, you will need a free \u003Ca href=\"https:\/\/www.dropbox.com\/\" target=\"_blank\"\u003EDropBox\u003C\/a\u003E or \u003Ca href=\"https:\/\/www.box.com\/\" target=\"_blank\"\u003Ebox\u003C\/a\u003E account and the three files I provide \u003Ca href=\"http:\/\/bit.ly\/L8rFnU\" target=\"_blank\"\u003Ehere\u003C\/a\u003E.\u0026nbsp;\u003Cspan style=\"background-color: white;\"\u003EThe lang-mma.js and prettify.css files I have provided for this post were forked from the\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;\u003C\/span\u003E\u003Ca href=\"http:\/\/meta.mathematica.stackexchange.com\/questions\/268\/including-google-code-prettify-language-extension-for-mathematica-into-this-site\" target=\"_blank\"\u003EMathematica - Stack Exchange\u003C\/a\u003E\u003Cspan style=\"background-color: white;\"\u003E.\u003C\/span\u003E\u003Cspan style=\"background-color: white;\"\u003E\u0026nbsp;Then follow these steps:\u003C\/span\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003C\/span\u003E\u003Cbr \/\u003E\u003Col\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003Cli\u003E\u003Cspan style=\"background-color: white;\"\u003EPlace the uncompressed files anywhere in your preferred cloud drive account.\u003C\/span\u003E\u003C\/li\u003E\u003Cbr \/\u003E\u003Cli\u003E\u003Cspan style=\"background-color: white;\"\u003EGet the sharable hyperlinks of those three files.\u003C\/span\u003E\u003C\/li\u003E\u003Cbr \/\u003E\u003Cli\u003E\u003Cspan style=\"background-color: white;\"\u003EIn Blogger (with the new 2012 interface) go to the Template tab on the left, then navigate to the Edit HTML button and proceed.\u003C\/span\u003E\u003C\/li\u003E\u003Cbr \/\u003E\u003Cli\u003E\u003Cspan style=\"background-color: white;\"\u003ECopy and paste the following code into your HTML template (best spot would be right before \u0026lt;\/head\u0026gt;) and insert the appropriate links in step 2 and save.\u003C\/span\u003E\u003C\/li\u003E\u003C\/span\u003E\u003C\/ol\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: white;\"\u003E\u003C\/span\u003E\u003Cbr \/\u003E\u003Cdiv\u003E\u003Csyntaxhighlighter class=\"brush: html\"\u003E\u0026lt;link href='INSERT LINK TO prettify.css' rel='stylesheet' type='text\/css'\/\u0026gt; \u0026lt;script language='javascript' src='INSERT LINK TO prettify.js' type='text\/javascript'\/\u0026gt; \u0026lt;script language='javascript' src='INSERT LINK TO lang-mma.js' type='text\/javascript'\/\u0026gt; \u0026lt;script type='text\/javascript'\u0026gt;     document.addEventListener('DOMContentLoaded',function() {         prettyPrint();     }); \u0026lt;\/script\u0026gt; \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EYou can now use the script to create Mathematica styled syntax in a Blogger post by using the tag name \"pre\" and the \"prettyprint\" class in the HTML code of a Blogger post. Try it out with the Mega Millions Mathematica code from above.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Csyntaxhighlighter class=\"brush: html\"\u003E\u0026lt;pre class=\"prettyprint lang-mma\"\u0026gt; \u0026lt;!--Paste Mathematica code here--\u0026gt; \u0026lt;\/pre\u0026gt; \u003C\/syntaxhighlighter\u003E\u003Cbr \/\u003EAs seen on his post for other languages, I prefer to use\u0026nbsp;\u003Ca href=\"http:\/\/alexgorbatchev.com\/SyntaxHighlighter\/\" style=\"background-color: white;\" target=\"_blank\"\u003ESyntaxHighlighter\u003C\/a\u003E\u0026nbsp;over google-code-prettify. The instructions on how to setup and use SyntaxHighlighter on Blogger can be found \u003Ca href=\"http:\/\/alexgorbatchev.com\/SyntaxHighlighter\/manual\/installation.html\" target=\"_blank\"\u003Ehere\u003C\/a\u003E, so I will not be covering it - unless requested. You can find all the files used for this post \u003Ca href=\"http:\/\/bit.ly\/L8rFnU\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. Happy Blogging Everyone!\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/5129865044444423769\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/simple-step-to-create-slicker-site.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/5129865044444423769"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/5129865044444423769"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/simple-step-to-create-slicker-site.html","title":"Simple Step to Create a Slicker Site"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-LpNteplfdSc\/T-NEd3DHTuI\/AAAAAAAAASQ\/aBL5Y46t9OE\/s72-c\/MegaMillions.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-1666655537606400825"},"published":{"$t":"2012-06-12T17:21:00.001-04:00"},"updated":{"$t":"2012-06-28T13:26:36.174-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Autoregressive modeling"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Crime Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Time-series analysis"}],"title":{"type":"text","$t":"First Attempt at Modeling Crime in Chicago, Part 2"},"content":{"type":"html","$t":"In the last \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in.html\" target=\"_blank\"\u003Epost\u003C\/a\u003E, my main goal was to develop a Mathematica 8 program that could regress coefficients for general autoregressive (AR) models. In my excitement, I skipped an essential step of data modeling. I forgot to check if the data could be accurately described by an AR model.\u003Cbr \/\u003E\u003Cbr \/\u003ETo keep things simple for the remainder of my attempts to model the number of daily crimes in Chicago, I will be using only the data between July 1st, 2011 and March 31st, 2012. This will also give me a reason to use the new \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/p\/data.html\" target=\"_blank\"\u003EData section\u003C\/a\u003E of my blog.\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-y10388H6KpU\/T9d0wVIWXTI\/AAAAAAAAAQ4\/5DrB_Dl8mec\/s1600\/CrimeTimeSeries.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"424\" src=\"http:\/\/4.bp.blogspot.com\/-y10388H6KpU\/T9d0wVIWXTI\/AAAAAAAAAQ4\/5DrB_Dl8mec\/s640\/CrimeTimeSeries.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EBefore examining the legitimacy of an AR model selection, I need to check if the data is stationary. The basis for any time series analysis is a stationary time series, because essentially we can only develop models and forecasts for stationary time series. In my experience, I have found no clear demarcation between stationary and non-stationary data. The usual approach to determine stationarity is to plot the autocorrelation function and if the plot doesn't dampen at long lags than the data is most likely not stationary. This logic escapes me. It may be because my statistical skills come from my engineering and computational chemistry training but I don't think the autocorrelation function is defined for non-stationary processes.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EOne of the tricks to transform a non-stationary dataset to a stationary one is to take the difference of the non-stationary dataset. So for me, a \u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Variogram\" target=\"_blank\"\u003Evariogram\u003C\/a\u003E is the more logical tool for determining stationarity. A variogram gives a ratio of the variance of differences some \u003Ci\u003Ek\u003C\/i\u003E\u0026nbsp;time units apart and the variance of the\u0026nbsp;differences only one time unit apart. If you look at the equations below, as \u003Ci\u003Ek\u003C\/i\u003E goes to infinity the difference between \u003Ci\u003Ek\u003C\/i\u003E lag and \u003Ci\u003Ek\u003C\/i\u003E+1 lag will eventually be the same. Ideally, you can conclude a dataset is stationary when the variogram shows a stable asymptote.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003Cspan style=\"font-size: large;\"\u003E\\[G_{k}=\\frac{V(z_{t+k}-z_{t})}{V(z_{t+1}-z_{t})},\\;k=1,2,3...\\]\u003C\/span\u003E\u003Cbr \/\u003E\u003Cdiv style=\"text-align: left;\"\u003Ewhere\u003C\/div\u003E\u003Cspan style=\"background-color: transparent; font-size: large;\"\u003E\\[V(z_{t+k}-z_{t})=\\frac{\\sum_{t=1}^{n-k}(d^{ k}_{t}-(n-k)^{-1}\\sum d^{ k}_{t})^2}{n-k-1}\\]\u003C\/span\u003E\u003Cbr \/\u003E\u003Cspan style=\"font-size: large;\"\u003E\\[d^{k}_{t}=z_{t+k}-z_{t}\\]\u003C\/span\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cbr \/\u003EIn Mathematica, a sample variogram can be calculated accordingly:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cspan style=\"background-color: transparent;\"\u003ELagDifferences[list_, k_Integer?Positive] \/; k \u0026lt; Length[list] :=\u0026nbsp;\u003C\/span\u003E\u003Cspan style=\"background-color: transparent;\"\u003EDrop[list, k] - Drop[list, -k];\u003C\/span\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"p1\"\u003EVariogram[list_, k_] := (#\/First@#) \u0026amp;@(Variance \/@ Table[LagDifferences[list, i], {i, k}]);\u003C\/div\u003E\u003Cbr \/\u003ENow we can finally move on to model selection. As a rule of thumb, an AR(\u003Ci\u003Ep\u003C\/i\u003E) model can adequately model a set of data if the autocorrelation function (Acf) plot looks like an infinitely damped exponential or sine wave that tails off and the partial autocorrelation function (PAcf) plot is cut off after \u003Ci\u003Ep\u003C\/i\u003E lags. Plots of the variogram, autocorrelation function, and partial autocorrelation function for the Chicago crime data and differences of the data are shown below.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\"\u003E \u003Ccolgroup span=\"1\" style=\"background: #ffffff;\"\u003E\u003C\/colgroup\u003E\u003Ccolgroup span=\"2\" style=\"background: #ffffff;\"\u003E  \u003C\/colgroup\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EVariogram\u003C\/th\u003E\u003C\/tr\u003E\u003C\/thead\u003E  \u003Ctbody\u003E\u003Ctr\u003E\u003Ctd\u003ECrime Series\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-IHoreDS3PFQ\/T9eq39NCSLI\/AAAAAAAAARE\/MOj_o37CCjI\/s1600\/CrimeSeries-variogram.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"128\" src=\"http:\/\/4.bp.blogspot.com\/-IHoreDS3PFQ\/T9eq39NCSLI\/AAAAAAAAARE\/MOj_o37CCjI\/s200\/CrimeSeries-variogram.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003E1st Difference\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-MZ4inUSG5Vw\/T9esfJilhEI\/AAAAAAAAARc\/8LVXwOIPXRU\/s1600\/Diff1CrimeSeries-variogram.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"128\" src=\"http:\/\/2.bp.blogspot.com\/-MZ4inUSG5Vw\/T9esfJilhEI\/AAAAAAAAARc\/8LVXwOIPXRU\/s200\/Diff1CrimeSeries-variogram.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003E2nd Difference\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-lWCKSBuPeRs\/T9esjZuJopI\/AAAAAAAAARk\/Uf0nWrJVghs\/s1600\/Diff2CrimeSeries-variogram.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"128\" src=\"http:\/\/2.bp.blogspot.com\/-lWCKSBuPeRs\/T9esjZuJopI\/AAAAAAAAARk\/Uf0nWrJVghs\/s200\/Diff2CrimeSeries-variogram.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E \u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\"\u003E \u003Ccolgroup span=\"1\" style=\"background: #ffffff;\"\u003E\u003C\/colgroup\u003E\u003Ccolgroup span=\"2\" style=\"background: #ffffff;\"\u003E  \u003C\/colgroup\u003E\u003Cthead\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EAutocorrelation\u003C\/th\u003E\u003Cth\u003EPartial Autocorrelation\u003C\/th\u003E\u003C\/tr\u003E\u003C\/thead\u003E  \u003Ctbody\u003E\u003Ctr\u003E\u003Ctd\u003ECrime Series\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-bKKoYp-hERI\/T9eq_CaQk8I\/AAAAAAAAARM\/YbT4l6MhhE0\/s1600\/CrimeSeries-acf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/1.bp.blogspot.com\/-bKKoYp-hERI\/T9eq_CaQk8I\/AAAAAAAAARM\/YbT4l6MhhE0\/s200\/CrimeSeries-acf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-S9-OXHy7uIM\/T9erLXTuqDI\/AAAAAAAAARU\/MuYyk5Q30zg\/s1600\/CrimeSeries-pacf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/4.bp.blogspot.com\/-S9-OXHy7uIM\/T9erLXTuqDI\/AAAAAAAAARU\/MuYyk5Q30zg\/s200\/CrimeSeries-pacf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003E1st Difference\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-aKCCqUzkd-k\/T9esqO9AM0I\/AAAAAAAAARs\/qyP2i_Iq8_Q\/s1600\/Diff1CrimeSeries-acf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/4.bp.blogspot.com\/-aKCCqUzkd-k\/T9esqO9AM0I\/AAAAAAAAARs\/qyP2i_Iq8_Q\/s200\/Diff1CrimeSeries-acf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-YL9ZuVENOT8\/T9eszJSTjZI\/AAAAAAAAAR0\/5kX9t9YUnvQ\/s1600\/Diff1CrimeSeries-pacf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/3.bp.blogspot.com\/-YL9ZuVENOT8\/T9eszJSTjZI\/AAAAAAAAAR0\/5kX9t9YUnvQ\/s200\/Diff1CrimeSeries-pacf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd\u003E2nd Difference\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-i7yTdWBu0Ho\/T9es4whv-_I\/AAAAAAAAAR8\/XMm-4qZnAIM\/s1600\/Diff2CrimeSeries-acf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/4.bp.blogspot.com\/-i7yTdWBu0Ho\/T9es4whv-_I\/AAAAAAAAAR8\/XMm-4qZnAIM\/s200\/Diff2CrimeSeries-acf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003Ctd\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-Jvr8onKzp88\/T9es_3IKwvI\/AAAAAAAAASE\/CbwRJ7SeGZc\/s1600\/Diff2CrimeSeries-pacf.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/3.bp.blogspot.com\/-Jvr8onKzp88\/T9es_3IKwvI\/AAAAAAAAASE\/CbwRJ7SeGZc\/s200\/Diff2CrimeSeries-pacf.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E \u003C\/table\u003E\u003Cbr \/\u003EIt looks like taking the first difference will produce a more stationary time series. The large negative autocorrelation at lag 1 in both the Acf and PAcf plot suggests the data may be better described with a moving average model component rather than with just an autoregressive model. Looks like it is bullocks on me for jumping the gun. I'll be back next post with a moving average model made in Mathematica. Until then, have fun with the source files for this post \u003Ca href=\"http:\/\/bit.ly\/LkCKTW\" target=\"_blank\"\u003Ehere\u003C\/a\u003E."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/1666655537606400825\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in_12.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1666655537606400825"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1666655537606400825"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in_12.html","title":"First Attempt at Modeling Crime in Chicago, Part 2"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/4.bp.blogspot.com\/-y10388H6KpU\/T9d0wVIWXTI\/AAAAAAAAAQ4\/5DrB_Dl8mec\/s72-c\/CrimeTimeSeries.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-379148330248674989"},"published":{"$t":"2012-06-05T18:56:00.003-04:00"},"updated":{"$t":"2012-06-05T18:56:38.685-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Autoregressive modeling"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Crime Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Socrata API"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Time-series analysis"}],"title":{"type":"text","$t":"First Attempt at Modeling Crime in Chicago"},"content":{"type":"html","$t":"In my last \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2012\/05\/chicago-crime-updated.html\" target=\"_blank\"\u003Epost\u003C\/a\u003E, I showed how to use the \u003Ca href=\"http:\/\/data.cityofchicago.org\/api\/docs\" target=\"_blank\"\u003ESocrata Open Data API\u003C\/a\u003E (SODA) to download the data of the crimes reported in Chicago, how to plot the locations, and how to count the number of crimes that occurred in a certain area. This post will chronicle my first attempts to model that data in Mathematica.\u003Cbr \/\u003E\u003Cbr \/\u003EThe time series analysis add-on to Mathematica 8 costs about $295. So I've decided to write my own autoregressive model (AR) package as a start to modeling the crimes in Chicago. My AR code in Mathematica is based off the \u003Ca href=\"http:\/\/cran.r-project.org\/web\/packages\/FitAR\/index.html\" target=\"_blank\"\u003EFitAR\u003C\/a\u003E R-package and its associated \u003Ca href=\"http:\/\/www.jstatsoft.org\/v28\/i02\/paper\" target=\"_blank\"\u003Epaper\u003C\/a\u003E. The mathematics and derivations of autoregressive models are already heavily covered on other \u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Autoregressive_model\" target=\"_blank\"\u003Ewebsites\u003C\/a\u003E,\u0026nbsp;so I will not be explaining it here. The alpha version of my code can be found \u003Ca href=\"http:\/\/bit.ly\/LxBFsp\" target=\"_blank\"\u003Ehere\u003C\/a\u003E, note it is not complete and does not yet have all the functions of FitAR or the Mathematica time series add-on.\u003Cbr \/\u003E\u003Cbr \/\u003EMy code, at this point, does provide fits that are comparable to the FitAR package. Using the default \"lynx\" data in R the following fits and associated residual autocorrelation plots were produced:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cstyle type=\"text\/css\"\u003E.mask{    position: relative;    overflow: hidden;    margin: 0px auto;    width: 100%;    background-color: #ffffff } .header{    float: left;    width: 100%;    background-color: #ffffff } .colleft{    position: relative;    width: 100%;    right: 50%;    background-color: #ffffff } .col1{    position: relative;    overflow: hidden;    float: left;    width: 48%;    left: 101%;    background-color: #ffffff } .col2{    position: relative;    overflow: hidden;    float: left;    width: 48%;    left: 3%;    background-color: #ffffff } .footer{    float: left;    width: 100%;    background-color: #ffffff } \u003C\/style\u003E\u003Cbr \/\u003E\u003Cdiv class=\"mask\"\u003E\u003Cdiv class=\"colleft\"\u003E\u003Cdiv class=\"col1\"\u003EMathematica\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" summary=\"Results from my Mathematica implementation of a AR(1) fit for the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem\u003EAR(1) Fit\u003C\/em\u003E\u003C\/caption\u003E \u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E0.71729\u003C\/td\u003E\u003Ctd\u003E0.0652589\u003C\/td\u003E\u003Ctd\u003E10.9914\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E1537.94\u003C\/td\u003E\u003Ctd\u003E364.477\u003C\/td\u003E\u003Ctd\u003E4.21957\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv style=\"text-align: left;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-4Bcyfvbud88\/T85hgIv5joI\/AAAAAAAAAOk\/8gJQ1VtYlgw\/s1600\/Residual+Auto+Mathematica+AR1.png\" imageanchor=\"1\" style=\"clear: left; margin-bottom: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/1.bp.blogspot.com\/-4Bcyfvbud88\/T85hgIv5joI\/AAAAAAAAAOk\/8gJQ1VtYlgw\/s200\/Residual+Auto+Mathematica+AR1.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" summary=\"Results from my Mathematica implementation of a AR(4) fit for the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003EAR(4) Fit\u003C\/em\u003E\u003C\/caption\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E1.12428\u003C\/td\u003E\u003Ctd\u003E0.0905874\u003C\/td\u003E\u003Ctd\u003E12.411\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(2)\u003C\/th\u003E\u003Ctd\u003E-0.716667\u003C\/td\u003E\u003Ctd\u003E0.136707\u003C\/td\u003E\u003Ctd\u003E-5.24237\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(3)\u003C\/th\u003E\u003Ctd\u003E0.262661\u003C\/td\u003E\u003Ctd\u003E0.136707\u003C\/td\u003E\u003Ctd\u003E1.92135\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(4)\u003C\/th\u003E\u003Ctd\u003E-0.253983\u003C\/td\u003E\u003Ctd\u003E0.0905874\u003C\/td\u003E\u003Ctd\u003E-2.80374\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E1537.94\u003C\/td\u003E\u003Ctd\u003E135.755\u003C\/td\u003E\u003Ctd\u003E11.3288\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-hvxToK213LM\/T85rx2rYiOI\/AAAAAAAAAPA\/DhR7eYjHq4M\/s1600\/Residual+Auto+Mathematica+AR4.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/1.bp.blogspot.com\/-hvxToK213LM\/T85rx2rYiOI\/AAAAAAAAAPA\/DhR7eYjHq4M\/s200\/Residual+Auto+Mathematica+AR4.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" summary=\"Results from my Mathematica implementation of subset AR fit for the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003ESubset AR(1,2,4,5,7,10,11) Fit\u003C\/em\u003E\u003C\/caption\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E0.820455\u003C\/td\u003E\u003Ctd\u003E0.0178649\u003C\/td\u003E\u003Ctd\u003E45.9256\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(2)\u003C\/th\u003E\u003Ctd\u003E-0.632818\u003C\/td\u003E\u003Ctd\u003E0.0972914\u003C\/td\u003E\u003Ctd\u003E-6.50436\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(4)\u003C\/th\u003E\u003Ctd\u003E-0.141951\u003C\/td\u003E\u003Ctd\u003E0.0684639\u003C\/td\u003E\u003Ctd\u003E-2.07337\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(5)\u003C\/th\u003E\u003Ctd\u003E0.141893\u003C\/td\u003E\u003Ctd\u003E0.0748193\u003C\/td\u003E\u003Ctd\u003E1.89647\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(7)\u003C\/th\u003E\u003Ctd\u003E0.202038\u003C\/td\u003E\u003Ctd\u003E0.0992876\u003C\/td\u003E\u003Ctd\u003E2.03488\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(10)\u003C\/th\u003E\u003Ctd\u003E-0.314105\u003C\/td\u003E\u003Ctd\u003E0.0917768\u003C\/td\u003E\u003Ctd\u003E-3.42249\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(11)\u003C\/th\u003E\u003Ctd\u003E-0.368658\u003C\/td\u003E\u003Ctd\u003E0.0870617\u003C\/td\u003E\u003Ctd\u003E-4.23444\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E6.68591\u003C\/td\u003E\u003Ctd\u003E0.100657\u003C\/td\u003E\u003Ctd\u003E66.4225\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-BNGsr1TqyIU\/T85wJdRtecI\/AAAAAAAAAPM\/XNNXIXPdGcs\/s1600\/Residual+Auto+Mathematica+AR.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"81\" src=\"http:\/\/3.bp.blogspot.com\/-BNGsr1TqyIU\/T85wJdRtecI\/AAAAAAAAAPM\/XNNXIXPdGcs\/s200\/Residual+Auto+Mathematica+AR.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cdiv class=\"col2\"\u003ER\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" summary=\"This is the R produced AR(1) fit of the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem\u003EAR(1) Fit\u003C\/em\u003E\u003C\/caption\u003E \u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E0.717303\u003C\/td\u003E\u003Ctd\u003E0.0652577\u003C\/td\u003E\u003Ctd\u003E10.9918\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E1538.02\u003C\/td\u003E\u003Ctd\u003E363.986\u003C\/td\u003E\u003Ctd\u003E4.22548\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv style=\"text-align: left;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-H2EfZ8R_uTU\/T85iFBV_-uI\/AAAAAAAAAOs\/yLv9KVM_xa0\/s1600\/Residual+AutoCorrelation-AR1.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"86\" src=\"http:\/\/1.bp.blogspot.com\/-H2EfZ8R_uTU\/T85iFBV_-uI\/AAAAAAAAAOs\/yLv9KVM_xa0\/s200\/Residual+AutoCorrelation-AR1.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" style=\"background-color: transparent; text-align: center;\" summary=\"This is the R produced AR(4) fit of the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003EAR(4) Fit\u003C\/em\u003E\u003C\/caption\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E1.12463\u003C\/td\u003E\u003Ctd\u003E0.0905803\u003C\/td\u003E\u003Ctd\u003E12.4158\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(2)\u003C\/th\u003E\u003Ctd\u003E-0.717396\u003C\/td\u003E\u003Ctd\u003E0.1367150\u003C\/td\u003E\u003Ctd\u003E-5.24738\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(3)\u003C\/th\u003E\u003Ctd\u003E0.263355\u003C\/td\u003E\u003Ctd\u003E0.136715\u003C\/td\u003E\u003Ctd\u003E1.92630\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(4)\u003C\/th\u003E\u003Ctd\u003E-0.254273\u003C\/td\u003E\u003Ctd\u003E0.0905803\u003C\/td\u003E\u003Ctd\u003E-2.80716\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E1538.02\u003C\/td\u003E\u003Ctd\u003E135.469\u003C\/td\u003E\u003Ctd\u003E11.3533\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-scZmnq_cFQg\/T85riIsJNJI\/AAAAAAAAAO4\/ophSJhBQywQ\/s1600\/Residual+AutoCorrelation-AR4.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"85\" src=\"http:\/\/4.bp.blogspot.com\/-scZmnq_cFQg\/T85riIsJNJI\/AAAAAAAAAO4\/ophSJhBQywQ\/s200\/Residual+AutoCorrelation-AR4.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Ctable border=\"1\" style=\"background-color: transparent; text-align: center;\" summary=\"This is the R produced subset AR fit of the lynx data.\"\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E \u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Ci\u003E\u003Cbr \/\u003E\u003C\/i\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Ci\u003E\u003Cbr \/\u003E\u003C\/i\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003E\u003Cbr \/\u003E\u003C\/em\u003E\u003C\/caption\u003E\u003Ccaption\u003E\u003Cem style=\"background-color: transparent;\"\u003ESubset AR(1,2,4,5,7,10,11) Fit\u003C\/em\u003E\u003C\/caption\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Cth\u003E\u003C\/th\u003E\u003Cth\u003EMLE\u003C\/th\u003E\u003Cth\u003Esd\u003C\/th\u003E\u003Cth\u003EZ-ratio\u003C\/th\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(1)\u003C\/th\u003E\u003Ctd\u003E0.8204490\u003C\/td\u003E\u003Ctd\u003E0.01786005\u003C\/td\u003E\u003Ctd\u003E45.937651\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(2)\u003C\/th\u003E\u003Ctd\u003E-0.6328433\u003C\/td\u003E\u003Ctd\u003E0.09731087\u003C\/td\u003E\u003Ctd\u003E-6.503316\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(4)\u003C\/th\u003E\u003Ctd\u003E-0.1420888\u003C\/td\u003E\u003Ctd\u003E0.06847123\u003C\/td\u003E\u003Ctd\u003E-2.075161\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(5)\u003C\/th\u003E\u003Ctd\u003E0.1421388\u003C\/td\u003E\u003Ctd\u003E0.07481409\u003C\/td\u003E\u003Ctd\u003E1.899894\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(7)\u003C\/th\u003E\u003Ctd\u003E0.2021250\u003C\/td\u003E\u003Ctd\u003E0.09927376\u003C\/td\u003E\u003Ctd\u003E2.036036\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(10)\u003C\/th\u003E\u003Ctd\u003E-0.3140954\u003C\/td\u003E\u003Ctd\u003E0.09178210\u003C\/td\u003E\u003Ctd\u003E-3.422186\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Ephi(11)\u003C\/th\u003E\u003Ctd\u003E-0.3686589\u003C\/td\u003E\u003Ctd\u003E0.08706171\u003C\/td\u003E\u003Ctd\u003E-4.234455\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Cth\u003Emu\u003C\/th\u003E\u003Ctd\u003E6.6859329\u003C\/td\u003E\u003Ctd\u003E0.09999207\u003C\/td\u003E\u003Ctd\u003E66.864631\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-8-QxFv1bWFQ\/T85wvWIhkNI\/AAAAAAAAAPU\/1e1-FG93U7Y\/s1600\/Residual+AutoCorrelation-AR.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"85\" src=\"http:\/\/3.bp.blogspot.com\/-8-QxFv1bWFQ\/T85wvWIhkNI\/AAAAAAAAAPU\/1e1-FG93U7Y\/s200\/Residual+AutoCorrelation-AR.png\" width=\"200\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003C\/div\u003E\u003Cbr \/\u003ENow that I know my code is correct, the next step will be figuring out if an autoregressive model can properly describe the crime data from Chicago. The lynx data from above is almost trivial when compared to the chaotic mess of crime data.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-P7HwGzD8ynA\/T86GffSUuNI\/AAAAAAAAAPg\/_FeFWs3a_Rg\/s1600\/TimeSeriesPlotLynx.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"155\" src=\"http:\/\/2.bp.blogspot.com\/-P7HwGzD8ynA\/T86GffSUuNI\/AAAAAAAAAPg\/_FeFWs3a_Rg\/s400\/TimeSeriesPlotLynx.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003ETime Series Plot of lynx data\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-0fU9tv_09bs\/T86Gn-gqouI\/AAAAAAAAAPo\/kTXcc8QvY3E\/s1600\/Chicago_Crime-ARTimeSeries.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"265\" src=\"http:\/\/2.bp.blogspot.com\/-0fU9tv_09bs\/T86Gn-gqouI\/AAAAAAAAAPo\/kTXcc8QvY3E\/s400\/Chicago_Crime-ARTimeSeries.png\" width=\"400\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EPlot of daily totals of crimes in Chicago.\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003EJust for fun, this is the residual autocorrelation of an AR(5) regressed with the crime series data from above:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-c8w9RGEaYFU\/T86Jr_LT8xI\/AAAAAAAAAP0\/b3rUsheI8u8\/s1600\/Chicago_Crime-ARTimeSeriesAR5.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"130\" src=\"http:\/\/1.bp.blogspot.com\/-c8w9RGEaYFU\/T86Jr_LT8xI\/AAAAAAAAAP0\/b3rUsheI8u8\/s320\/Chicago_Crime-ARTimeSeriesAR5.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ENot a model anyone should ever depend on. The files used in this post can be found \u003Ca href=\"http:\/\/bit.ly\/LxBFsp\" target=\"_blank\"\u003Ehere\u003C\/a\u003E."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/379148330248674989\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/379148330248674989"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/379148330248674989"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/06\/first-attempt-at-modeling-crime-in.html","title":"First Attempt at Modeling Crime in Chicago"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-4Bcyfvbud88\/T85hgIv5joI\/AAAAAAAAAOk\/8gJQ1VtYlgw\/s72-c\/Residual+Auto+Mathematica+AR1.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-6212998535996863581"},"published":{"$t":"2012-05-07T14:00:00.002-04:00"},"updated":{"$t":"2012-05-07T14:00:54.049-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago Neighborhood Crime"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Crime Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"GIS"},{"scheme":"http://www.blogger.com/atom/ns#","term":"latitude"},{"scheme":"http://www.blogger.com/atom/ns#","term":"longitude"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Socrata API"}],"title":{"type":"text","$t":"Chicago Crime - Updated"},"content":{"type":"html","$t":"I was a bit dismayed to discover that the EveryBlock API stopped working. One of my most popular posts,\u0026nbsp;\u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/05\/chicago-crime-in-24-hours.html\" target=\"_blank\"\u003EChicago Crime\u003C\/a\u003E, depended on this API heavily.\u003Cbr \/\u003E\u003Cbr \/\u003EHere I present an alternative data source for my previous \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/05\/chicago-crime-in-24-hours.html\" target=\"_blank\"\u003Epost\u003C\/a\u003E and a fun update. The update is a quick and dirty method to create popular crime density plots, as shown below.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-5uNEjMAoLUk\/T6gM6zcWA3I\/AAAAAAAAAOM\/HAdKPaNSRiI\/s1600\/Chicago_Crime_DensityPoints.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-5uNEjMAoLUk\/T6gM6zcWA3I\/AAAAAAAAAOM\/HAdKPaNSRiI\/s1600\/Chicago_Crime_DensityPoints.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003EIn the spirit of\u0026nbsp;transparency, the city of Chicago has taken the initiative to electronically release a great deal of data with \u003Ca href=\"http:\/\/data.cityofchicago.org\/api\/docs\" target=\"_blank\"\u003ESocrata Open Data API\u003C\/a\u003E (SODA). Luckily for us, this release includes all filed police reports. The records available through this API range from the year 2001 to the present, which dwarfs the two week range provided by\u0026nbsp;EveryBlock's API. However, I would not recommend downloading all of this data as the text file is approximately 2GB in size.\u003Cbr \/\u003E\u003Cbr \/\u003EIn this post I use only the filed police reports from one year prior to present. The data is available in both JSON and CSV format. I only use the CSV format here for the convenience of those with an earlier version of Mathematica. To import this data into Mathematica:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-uxqHgJfSx8E\/T6f4eNQNJjI\/AAAAAAAAAMo\/U5bpmavsVns\/s1600\/Chicago_Crime_csv.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-uxqHgJfSx8E\/T6f4eNQNJjI\/AAAAAAAAAMo\/U5bpmavsVns\/s1600\/Chicago_Crime_csv.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EA plot of all reported crimes in Chicago between\u0026nbsp;February 1st and March 31st of 2012 was created with the data.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-wdfk8kp5X2M\/T6fvnney4bI\/AAAAAAAAAMM\/G2UCj32tIrU\/s1600\/Chicago_Crime_Plot.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-wdfk8kp5X2M\/T6fvnney4bI\/AAAAAAAAAMM\/G2UCj32tIrU\/s1600\/Chicago_Crime_Plot.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EPlot of incidents for any period within the year can be easily created with the following Mathematica code. Restrict the data to the range of interest by replacing the dates of the start and end strings. Dates must be in the format of mm\/dd\/yyyy. This code simply finds the positions of the strings that match the inputed dates and plots the daily totals of incidents.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-H_Jmi-5-Mck\/T6f7bFj8VzI\/AAAAAAAAANA\/GDCWo5WgZ0E\/s1600\/Chicago_Crime_AllPlot.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-H_Jmi-5-Mck\/T6f7bFj8VzI\/AAAAAAAAANA\/GDCWo5WgZ0E\/s1600\/Chicago_Crime_AllPlot.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EThe data can be parsed into primary descriptions including arson, homicide, sex offense, weapons violation, crim sexual assault, interfere with public officer, gambling, public peace violation, criminal trespass, liquor law violation, assault, deceptive practice, burglary, criminal damage, robbery, motor vehicle theft, offense involving children, narcotics, theft, other offense, and battery. Below is the plot and code to filter for only narcotics incidents.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-lRnLTqQ38_g\/T6f8T570-AI\/AAAAAAAAANI\/gWjIXTTwkto\/s1600\/primary.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-lRnLTqQ38_g\/T6f8T570-AI\/AAAAAAAAANI\/gWjIXTTwkto\/s1600\/primary.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-Ptdo4cjOT3U\/T6f_--Ml3kI\/AAAAAAAAANs\/z0LcVJ57NE8\/s1600\/Chicago_Crime_Prime.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"292\" src=\"http:\/\/3.bp.blogspot.com\/-Ptdo4cjOT3U\/T6f_--Ml3kI\/AAAAAAAAANs\/z0LcVJ57NE8\/s640\/Chicago_Crime_Prime.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EA secondary description can also be used to parse the data. Some secondary descriptions include aggravated: handgun, poss: crack, armed: handgun, harassment by telephone, poss: heroin(white), to land, retail theft, unlawful entry, strongarm - no weapon, to property, poss: cannabis 30gms or less, from building, $500 and under, forcible entry, to vehicle, domestic battery simple, simple, automobile, over $500, telephone threat, and wireroom\/sports.\u0026nbsp;Below is the plot and code to filter for incidents with a primary description of narcotics and a secondary description of\u0026nbsp;poss: cannabis 30gms or less.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-vtaCASY7q40\/T6f8ZDVciiI\/AAAAAAAAANQ\/2yDt3qa9kJY\/s1600\/secondary.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-vtaCASY7q40\/T6f8ZDVciiI\/AAAAAAAAANQ\/2yDt3qa9kJY\/s1600\/secondary.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-K7O04OuLMZM\/T6gBHsPLCfI\/AAAAAAAAAN0\/XOpvlAyuECw\/s1600\/Chicago_Crime_Second.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"292\" src=\"http:\/\/3.bp.blogspot.com\/-K7O04OuLMZM\/T6gBHsPLCfI\/AAAAAAAAAN0\/XOpvlAyuECw\/s640\/Chicago_Crime_Second.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EThe incidents may also be separated into ones where arrests were made and ones where no arrests were made.\u0026nbsp;Below is code to filter for incidents with a primary description of narcotics and a secondary description of\u0026nbsp;poss: cannabis 30gms or less with arrests and without. Both plots comparing to total narcotics incidents are shown.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-ae6lWutdUpU\/T6f8eng6DEI\/AAAAAAAAANY\/C238gRX-fxA\/s1600\/arrests.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-ae6lWutdUpU\/T6f8eng6DEI\/AAAAAAAAANY\/C238gRX-fxA\/s1600\/arrests.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-qzDhP473ie8\/T6gD0_AkyoI\/AAAAAAAAAOA\/OOqx1s_xJlM\/s1600\/Chicago_Crime_Arrests.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"292\" src=\"http:\/\/2.bp.blogspot.com\/-qzDhP473ie8\/T6gD0_AkyoI\/AAAAAAAAAOA\/OOqx1s_xJlM\/s640\/Chicago_Crime_Arrests.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EThe method to display where certain incidents occur in chicago is similar to my previous post,\u0026nbsp;\u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/05\/chicago-crime-in-24-hours.html\" target=\"_blank\"\u003EChicago Crime\u003C\/a\u003E.\u0026nbsp;Please refer to that posting for a more detailed\u0026nbsp;explanation on how to create the plot below. The plot here is a geographical visualization of where arrests occurred relating to incidents of narcotics with possession of cannabis with 30 grams or less.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-SJ_qvcDPS7c\/T6fwKwcZYSI\/AAAAAAAAAMU\/WXHTSVgBlTg\/s1600\/Chicago_Crime_GIS.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-SJ_qvcDPS7c\/T6fwKwcZYSI\/AAAAAAAAAMU\/WXHTSVgBlTg\/s1600\/Chicago_Crime_GIS.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThe method to tabulate the number of incidents within neighborhoods is also similar to my\u0026nbsp;previous post,\u0026nbsp;\u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/05\/chicago-crime-in-24-hours.html\" target=\"_blank\"\u003EChicago Crime\u003C\/a\u003E.\u0026nbsp;This calculation totals the numbers of incidents within each neighborhood boundary. It\u0026nbsp;is the most costly in computational time and power because of the generality in the counting code that allows it to use any boundaries (such as census tracts, police beats, etc.) The tabulation of arrests made relating to incidents\u0026nbsp;of narcotics with possession of cannabis with 30 grams or less in each neighborhood is presented.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-iUpBacCRQSw\/T6fxZkWVVPI\/AAAAAAAAAMc\/4y_elhahwGg\/s1600\/Chicago_Crime_Count.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-iUpBacCRQSw\/T6fxZkWVVPI\/AAAAAAAAAMc\/4y_elhahwGg\/s1600\/Chicago_Crime_Count.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EWhile doing some initial research for this post, I found \"crime hotspots\" as a popular visualization option. A similar visualization is easily created in mathematica with three simple lines of code.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-VNzn_h80XHc\/T6f8-vf5yoI\/AAAAAAAAANg\/ntRgBqXDhcI\/s1600\/Chicago_Crime_DensityCode.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-VNzn_h80XHc\/T6f8-vf5yoI\/AAAAAAAAANg\/ntRgBqXDhcI\/s1600\/Chicago_Crime_DensityCode.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-MMlxWH79G-U\/T6fuLcMRzmI\/AAAAAAAAAME\/1sQdNLcbvcg\/s1600\/Chicago_Crime_Density.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"320\" src=\"http:\/\/2.bp.blogspot.com\/-MMlxWH79G-U\/T6fuLcMRzmI\/AAAAAAAAAME\/1sQdNLcbvcg\/s320\/Chicago_Crime_Density.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThe Mathematica code used in this post can be downloaded \u003Ca href=\"http:\/\/bit.ly\/KAeu4H\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. Some fun analysis of crimes in chicago will follow in future posts."},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/6212998535996863581\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/05\/chicago-crime-updated.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/6212998535996863581"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/6212998535996863581"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/05\/chicago-crime-updated.html","title":"Chicago Crime - Updated"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/3.bp.blogspot.com\/-5uNEjMAoLUk\/T6gM6zcWA3I\/AAAAAAAAAOM\/HAdKPaNSRiI\/s72-c\/Chicago_Crime_DensityPoints.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-3537084872757420682"},"published":{"$t":"2012-04-19T13:06:00.000-04:00"},"updated":{"$t":"2012-04-19T13:07:02.882-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Network Analysis"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Wikipedia"}],"title":{"type":"text","$t":"Can I Trust Wikipedia?"},"content":{"type":"html","$t":"A question as dreaded as \"Who's on first?\" And one that I am forced to ask as I feel lucky often. One quick and fun way to address how much you can trust a wikipedia article is by looking at what else a contributor edits. This can easily be done through the magic of the \u003Ca href=\"https:\/\/www.mediawiki.org\/wiki\/API\" target=\"_blank\"\u003EMediaWiki\u003C\/a\u003E API and \u003Ca href=\"http:\/\/www.wolfram.com\/mathematica\/\" target=\"_blank\"\u003EMathematica\u003C\/a\u003E. Here is the result, with a\u0026nbsp;\u003Ca href=\"http:\/\/www.cnn.com\/\" target=\"_blank\"\u003ECNN\u003C\/a\u003E inspired example, of Mitt Romney's wikipedia \u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Mitt_Romney\" target=\"_blank\"\u003Earticle\u003C\/a\u003E\u0026nbsp;from April 19th, 2012.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-qKVzsDAE3r4\/T5BEoJoFTrI\/AAAAAAAAALw\/uRNtEU_05is\/s1600\/mitt+romney+wikipedia+analysis.jpg\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" height=\"498\" src=\"http:\/\/2.bp.blogspot.com\/-qKVzsDAE3r4\/T5BEoJoFTrI\/AAAAAAAAALw\/uRNtEU_05is\/s640\/mitt+romney+wikipedia+analysis.jpg\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EThe top 20 contributors to the Mitt Romney wikipedia article are highlighted in yellow and common articles edited by these contributors are highlighted in blue.\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003EPersonally, I see red flags when contributors of the Mitt Romney article are also subject-matter experts of\u0026nbsp;\u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Katy_Perry\" target=\"_blank\"\u003EKaty Perry\u003C\/a\u003E,\u0026nbsp;\u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Springfield_(The_Simpsons)\" target=\"_blank\"\u003ESpringfield (The Simpsons)\u003C\/a\u003E, and\u0026nbsp;\u003Ca href=\"http:\/\/en.wikipedia.org\/wiki\/Savage_Love\" target=\"_blank\"\u003ESavage Love\u003C\/a\u003E. I wouldn't put too much faith in its accuracy.\u003Cbr \/\u003E\u003Cbr \/\u003EA copy of the notebook can be downloaded\u0026nbsp;\u003Ca href=\"http:\/\/bit.ly\/I5gcI5\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. To use it, simply copy the title of the wikipedia article of interest and paste it into the third line. More to come the next time I skip lunch!"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/3537084872757420682\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/04\/can-i-trust-wikipedia.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3537084872757420682"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3537084872757420682"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2012\/04\/can-i-trust-wikipedia.html","title":"Can I Trust Wikipedia?"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/2.bp.blogspot.com\/-qKVzsDAE3r4\/T5BEoJoFTrI\/AAAAAAAAALw\/uRNtEU_05is\/s72-c\/mitt+romney+wikipedia+analysis.jpg","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-1217990038597383797"},"published":{"$t":"2011-11-26T11:32:00.001-05:00"},"updated":{"$t":"2011-12-16T11:33:52.001-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Census"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Chicago"},{"scheme":"http://www.blogger.com/atom/ns#","term":"GIS"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"}],"title":{"type":"text","$t":"Making Sense of Census-less Data"},"content":{"type":"html","$t":"\u003Cdiv class=\"separator\" style=\"clear: both; text-align: left;\"\u003E\u003Cspan style=\"text-align: -webkit-auto;\"\u003EI have always found that\u0026nbsp;supplemental\u0026nbsp;3D visualizations help provide better insights on census data. While playing with the compiled data from the 2000 census, from\u0026nbsp;\u003C\/span\u003E\u003Ca href=\"http:\/\/www.cityofchicago.org\/content\/city\/en\/depts\/doit\/supp_info\/census_maps.html\" style=\"text-align: -webkit-auto;\" target=\"_blank\"\u003Ecityofchicago.org\u003C\/a\u003E\u003Cspan style=\"text-align: -webkit-auto;\"\u003E, I found an interesting anomaly.\u003C\/span\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-ts7q80f_BnE\/TtPXDlDSRDI\/AAAAAAAAAKc\/Uqtu1Kpn7CM\/s1600\/Income_MedianHousehold.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em; text-align: center;\"\u003E\u003Cimg border=\"0\" height=\"640\" src=\"http:\/\/3.bp.blogspot.com\/-ts7q80f_BnE\/TtPXDlDSRDI\/AAAAAAAAAKc\/Uqtu1Kpn7CM\/s640\/Income_MedianHousehold.png\" width=\"488\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-oNJtCJMXM8o\/TtPXM9IWLLI\/AAAAAAAAAKk\/cHuRfZDES7w\/s1600\/Chicago3Dmap.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em; text-align: center;\"\u003E\u003Cimg border=\"0\" height=\"595\" src=\"http:\/\/1.bp.blogspot.com\/-oNJtCJMXM8o\/TtPXM9IWLLI\/AAAAAAAAAKk\/cHuRfZDES7w\/s640\/Chicago3Dmap.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EAfter some investigation, I found the red spike represents 15 families with a median income of $255,000. Was there mischief afoot by those 15 families?\u003Cbr \/\u003E\u003Cbr \/\u003EI'll let the more curious readers decide. Attached \u003Ca href=\"http:\/\/bit.ly\/vLIhTY\" target=\"_blank\"\u003Ehere\u003C\/a\u003E is the census data and a Mathematica notebook. The notebook has been generalized to allow users to create 3D visualizations of all provided census data.\u003Cbr \/\u003E\u003Cbr \/\u003EDownload \u003Ca href=\"http:\/\/bit.ly\/vLIhTY\" target=\"_blank\"\u003Ehere\u003C\/a\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/1217990038597383797\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/making-sense-of-census-less-data.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1217990038597383797"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/1217990038597383797"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/making-sense-of-census-less-data.html","title":"Making Sense of Census-less Data"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/3.bp.blogspot.com\/-ts7q80f_BnE\/TtPXDlDSRDI\/AAAAAAAAAKc\/Uqtu1Kpn7CM\/s72-c\/Income_MedianHousehold.png","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-6528948377230243848"},"published":{"$t":"2011-11-10T14:17:00.000-05:00"},"updated":{"$t":"2011-11-10T15:18:38.979-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"latex"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Letterhead"},{"scheme":"http://www.blogger.com/atom/ns#","term":"pstricks"},{"scheme":"http://www.blogger.com/atom/ns#","term":"ScribTeX"}],"title":{"type":"text","$t":"Batman Letterhead in ScribTeX"},"content":{"type":"html","$t":"As soon as I made my last post, I got a few emails asking how to use LaTeX to create the Batman letterhead.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s1600\/Batman_Letterhead.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s1600\/Batman_Letterhead.jpg\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThrough the magic of \"the cloud,\" there is an easy way to play with the letterhead I created without downloading LaTeX. First, download the source file, \u003Ca href=\"http:\/\/bit.ly\/rY6OFc\" target=\"_blank\"\u003Ehere\u003C\/a\u003E.\u003Cbr \/\u003E\u003Cbr \/\u003EThere is a great website called \u003Ca href=\"http:\/\/www.scribtex.com\/\" target=\"_blank\"\u003EScribTeX\u003C\/a\u003E that allows the compilation of TeX files through the website. Signing up is painless and free. Once you have signed up and logged on, you will see the Dashboard page. Click on New Project and you will be asked to name the new project. I chose the name \"Batman Letterhead.\"\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-gtWK2d9ny_E\/Trwq5iNaqiI\/AAAAAAAAAJs\/Sb58K1wnfCI\/s1600\/new+proj.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"360\" src=\"http:\/\/1.bp.blogspot.com\/-gtWK2d9ny_E\/Trwq5iNaqiI\/AAAAAAAAAJs\/Sb58K1wnfCI\/s640\/new+proj.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThen upload the TeX file containing the letterhead.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-GGAxWb1i-7o\/TrwrAIZceNI\/AAAAAAAAAJ0\/PkVcJmJOxY0\/s1600\/upload.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"362\" src=\"http:\/\/3.bp.blogspot.com\/-GGAxWb1i-7o\/TrwrAIZceNI\/AAAAAAAAAJ0\/PkVcJmJOxY0\/s640\/upload.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EFor the TeX file to compile properly, you must change the compiler to\u0026nbsp;\u003Ci\u003Elatex\u003C\/i\u003E\u0026nbsp;from ScribTeX's default\u0026nbsp;\u003Ci\u003Epdflatex\u003C\/i\u003E\u0026nbsp;compiler. The\u0026nbsp;\u003Ci\u003Epdflatex\u003C\/i\u003E\u0026nbsp;compiler compiles directly to a pdf. The\u0026nbsp;\u003Ci\u003Elatex\u003C\/i\u003E\u0026nbsp;compiler will create a postscript file before creating a pdf. The packages used to plot the Batman logo (pstricks and pst-plot) only make sense to postscript files and cannot be used with\u0026nbsp;\u003Ci\u003Epdflatex\u003C\/i\u003E. To change the compiler, click on \"Settings\" from the project page then the \"Compiler Settings\" tab. \u0026nbsp;After choosing\u0026nbsp;\u003Ci\u003Elatex\u003C\/i\u003E, click \"Save\" then \"Files.\"\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-WxMpvHDCtMw\/TrwrH4jxDPI\/AAAAAAAAAJ8\/N7NBn7x7Fw8\/s1600\/compiler.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"362\" src=\"http:\/\/4.bp.blogspot.com\/-WxMpvHDCtMw\/TrwrH4jxDPI\/AAAAAAAAAJ8\/N7NBn7x7Fw8\/s640\/compiler.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ENow open the uploaded TeX file and you will see that it is the same file as my previous post with one major difference. Instead of using a separate file containing the data points, this one has the data points stored within.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-OPdfWUxIJcI\/TrwrPJY7snI\/AAAAAAAAAKE\/bVrMBkLt49k\/s1600\/changedtex.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"362\" src=\"http:\/\/3.bp.blogspot.com\/-OPdfWUxIJcI\/TrwrPJY7snI\/AAAAAAAAAKE\/bVrMBkLt49k\/s640\/changedtex.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003ETo compile the letterhead, click \"Compile\" and a new window will open with the compiled PDF.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-HgfCFtKAvHU\/Trwrjz5-s8I\/AAAAAAAAAKM\/WdGyV0dzVx0\/s1600\/compiledpdf.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"362\" src=\"http:\/\/3.bp.blogspot.com\/-HgfCFtKAvHU\/Trwrjz5-s8I\/AAAAAAAAAKM\/WdGyV0dzVx0\/s640\/compiledpdf.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003EThe TeX file used in this post can be found\u0026nbsp;\u003Ca href=\"http:\/\/bit.ly\/rY6OFc\" target=\"_blank\"\u003Ehere\u003C\/a\u003E. Enjoy TeXing as Batman!"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/6528948377230243848\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/batman-letterhead-in-scribtex.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/6528948377230243848"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/6528948377230243848"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/batman-letterhead-in-scribtex.html","title":"Batman Letterhead in ScribTeX"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s72-c\/Batman_Letterhead.jpg","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-3340076511374902305"},"published":{"$t":"2011-11-10T13:04:00.001-05:00"},"updated":{"$t":"2011-11-10T13:05:40.515-05:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"Batman"},{"scheme":"http://www.blogger.com/atom/ns#","term":"latex"},{"scheme":"http://www.blogger.com/atom/ns#","term":"Letterhead"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"plotting"}],"title":{"type":"text","$t":"Batman Letterhead"},"content":{"type":"html","$t":"After reading a post from \u003Ca href=\"http:\/\/www.hardocp.com\/news\/2011\/07\/29\/batman_equation\/\" target=\"_blank\"\u003EHardOCP\u003C\/a\u003E that gave the roots of the Batman symbol, I could not help imagining the letterhead of the world's greatest detective. The letterhead needs to say \"Batman is classy.\"\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s1600\/Batman_Letterhead.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s1600\/Batman_Letterhead.jpg\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003EI created this letterhead using the \"Batman equation\", Mathematica, and LaTeX. The computer in the Bat Cave is probably Unix-based and cannot use Microsoft Word. Therefore, Batman must use LaTeX. The first step to creating this letterhead is to plot the symbol. A post from \u003Ca href=\"http:\/\/playingwithmathematica.com\/2011\/08\/06\/the-batman-curve\/\" target=\"_blank\"\u003EPlaying With Mathematica\u003C\/a\u003E has done this for me already with seven simple lines.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-cJ4XytcUlnM\/TrwGBr9turI\/AAAAAAAAAIM\/vu0W16n1JJs\/s1600\/BatmanEquations.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"87\" src=\"http:\/\/2.bp.blogspot.com\/-cJ4XytcUlnM\/TrwGBr9turI\/AAAAAAAAAIM\/vu0W16n1JJs\/s640\/BatmanEquations.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThe Show function is used to overlay the six plots to reveal the desired product.\u003Cbr \/\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-mjdS9YnDqbs\/TrwGZLm9_ZI\/AAAAAAAAAIU\/WO3D1Nufzh4\/s1600\/BatmanPlot.png\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"320\" src=\"http:\/\/2.bp.blogspot.com\/-mjdS9YnDqbs\/TrwGZLm9_ZI\/AAAAAAAAAIU\/WO3D1Nufzh4\/s320\/BatmanPlot.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003ETo extract the points used for the final plot the following command must be executed for each root equation. \u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-85iFOiG6Mg8\/TrwLpU25ORI\/AAAAAAAAAIk\/34l3GXS2FuE\/s1600\/Batman-extractplot.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"23\" src=\"http:\/\/2.bp.blogspot.com\/-85iFOiG6Mg8\/TrwLpU25ORI\/AAAAAAAAAIk\/34l3GXS2FuE\/s320\/Batman-extractplot.png\" width=\"320\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003EAfter extracting and compiling all data points into one .dat file, we are ready to create the letterhead in LaTeX. The following LaTeX code relies on the pstricks and pst-plot packages. It plots the points in the saved data file and creates the bar that separates Batman's contact information.\u003Cbr \/\u003E\u003Cbr \/\u003E\\begin{document}\u003Cbr \/\u003E\\readdata{\\BatData}{BatData.dat}\u003Cbr \/\u003E\\noindent\\begin{minipage}[b]{5cm}\u003Cbr \/\u003E\\begin{center}\u003Cbr \/\u003E\\psset{xunit=0.35cm,yunit=0.5cm}\u003Cbr \/\u003E\\begin{pspicture}(0,0)(0,2)\u003Cbr \/\u003E\\listplot[plotstyle=dots]{\\BatData}\u003Cbr \/\u003E\\end{pspicture}\u003Cbr \/\u003E\\end{center}\u003Cbr \/\u003E\\end{minipage}\u003Cbr \/\u003E\\hfill\u003Cbr \/\u003E\\begin{minipage}[b]{7cm}\u003Cbr \/\u003E\\begin{flushright}\u003Cbr \/\u003E\\footnotesize{\\itshape 1007 Mountain Drive {\\scriptsize$\\bullet$} Gotham City, NY 10027}\u003Cbr \/\u003E\\end{flushright}\u003Cbr \/\u003E\\end{minipage}\u003Cbr \/\u003E\\vskip-2mm\u003Cbr \/\u003E{\\hspace{40mm}\\hfill\\rule[0.5mm]{130mm}{0.5pt}}\u003Cbr \/\u003E\\vskip-2mm\u003Cbr \/\u003E\\hfill\u003Cbr \/\u003E\\begin{minipage}[b]{7cm}\u003Cbr \/\u003E\\begin{flushright}\u003Cbr \/\u003E\\footnotesize{\\itshape 555.555.5555 {\\scriptsize$\\bullet$} \\href{mailto:Bruce@WayneCorp.com}{Bruce@WayneCorp.com}}\u003Cbr \/\u003E\\end{flushright}\u003Cbr \/\u003E\\end{minipage}\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EAll of the files discussed in this post can be downloaded \u003Ca href=\"http:\/\/bit.ly\/sUUXDA\" target=\"_blank\"\u003Ehere\u003C\/a\u003E, along\u0026nbsp;with a\u0026nbsp;\u003Ca href=\"http:\/\/bit.ly\/t7xLHk\" target=\"_blank\"\u003Eletter\u003C\/a\u003E\u0026nbsp;Batman would've sent to The Joker.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-vEVdelsSDos\/TrwOznMHySI\/AAAAAAAAAIs\/5oDUokpxQ_8\/s1600\/Batman_Letter.jpg\" imageanchor=\"1\" style=\"margin-left: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-vEVdelsSDos\/TrwOznMHySI\/AAAAAAAAAIs\/5oDUokpxQ_8\/s1600\/Batman_Letter.jpg\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/3340076511374902305\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/batman-letterhead.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3340076511374902305"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/3340076511374902305"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/batman-letterhead.html","title":"Batman Letterhead"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/4.bp.blogspot.com\/-plX5LxNzSIM\/Trv-pE26BEI\/AAAAAAAAAIE\/3Kh-lvP9ot8\/s72-c\/Batman_Letterhead.jpg","height":"72","width":"72"},"thr$total":{"$t":"0"}},{"id":{"$t":"tag:blogger.com,1999:blog-6855656749413817524.post-4497388060004690940"},"published":{"$t":"2011-11-08T22:08:00.001-05:00"},"updated":{"$t":"2012-06-21T13:38:13.898-04:00"},"category":[{"scheme":"http://www.blogger.com/atom/ns#","term":"google insights"},{"scheme":"http://www.blogger.com/atom/ns#","term":"mathematica"},{"scheme":"http://www.blogger.com/atom/ns#","term":"parsing"},{"scheme":"http://www.blogger.com/atom/ns#","term":"python"},{"scheme":"http://www.blogger.com/atom/ns#","term":"pythonika"},{"scheme":"http://www.blogger.com/atom/ns#","term":"stock trends"}],"title":{"type":"text","$t":"Google Insight Data to Predict Stock Market Trends, Part 1"},"content":{"type":"html","$t":"Previously, I have demonstrated \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/installing-pythonika.html\" target=\"_blank\"\u003Ehow\u003C\/a\u003E to install pythonika and \u003Ca href=\"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/after-reading-web-search-queries-can.html\" target=\"_blank\"\u003Ewhy\u003C\/a\u003E weekends need to be considered in analyzing market trends. \u0026nbsp;Here I will share my code on importing Google Insights data into Mathematica and provide a neat example of how to parse and analyze the data.\u003Cbr \/\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003ETo get started, please install pythonika and download my \u003Ca href=\"http:\/\/bit.ly\/Mwg7iq\" target=\"_blank\"\u003Esample notebook\u003C\/a\u003E. \u0026nbsp;Change the first line to point to your compiled executable of pythonika.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-n-PbQnpMEzU\/Trm7OXHjefI\/AAAAAAAAAGc\/CYIkOhhdQBE\/s1600\/path.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/1.bp.blogspot.com\/-n-PbQnpMEzU\/Trm7OXHjefI\/AAAAAAAAAGc\/CYIkOhhdQBE\/s1600\/path.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThen replace \"USERNAME\" and \"PASSWORD\" in the following line with your google account.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-0qXHKO-fFAU\/Trm7qSVE-vI\/AAAAAAAAAGk\/pSWXX41Hxu4\/s1600\/login.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-0qXHKO-fFAU\/Trm7qSVE-vI\/AAAAAAAAAGk\/pSWXX41Hxu4\/s1600\/login.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003EEvaluating the python cell, will download the daily search data of \"AAPL\" in the last 90 days. \u0026nbsp;To limit the search to the last 30 days, the date variable must be set to \"today 01-m.\" \u0026nbsp;An important note, any data beyond 90 days are averaged over a week and will need to be parsed in a manner different from the example here.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThis command will parse the CSV data for the dates and total daily searches:\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-TvfxIAsHDvg\/Trm_WJ3yWUI\/AAAAAAAAAGs\/jS-Yl1Nmjg0\/s1600\/parse.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"19\" src=\"http:\/\/4.bp.blogspot.com\/-TvfxIAsHDvg\/Trm_WJ3yWUI\/AAAAAAAAAGs\/jS-Yl1Nmjg0\/s640\/parse.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003ESince I am interested in the financials of Apple (AAPL), specifically its daily traded volume, I have set :\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-W-diy6LKlxk\/TrnB4Rx-N5I\/AAAAAAAAAG0\/3z6YouwUNDE\/s1600\/financedata.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-W-diy6LKlxk\/TrnB4Rx-N5I\/AAAAAAAAAG0\/3z6YouwUNDE\/s1600\/financedata.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003EThe variables googstartdate and googenddate are automatically set to the date range indicated in the first cell. \u0026nbsp;The next part of the code is used to pair up trading dates with query dates, filter out the weekends, and remove lines with missing data.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThere may be a more efficient method of matching two lists of unmatched lengths in Mathematica, but this was the first one I thought up that could account for holidays. Essentially, I have created a set of {dates, queries} and another set of {dates, stock volume} and must match the dates up to create one set of {dates, queries, stock volume}. \u0026nbsp;First I joined the lists together, then I reordered the two lists by:\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-EZmtw2HZvVg\/TrnHeO584EI\/AAAAAAAAAG8\/R_eTmfPvJ9w\/s1600\/order.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"24\" src=\"http:\/\/4.bp.blogspot.com\/-EZmtw2HZvVg\/TrnHeO584EI\/AAAAAAAAAG8\/R_eTmfPvJ9w\/s640\/order.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThe new set will contain disparities from weekends and possibly blank results from the Google Insight data. \u0026nbsp;To filter out these disparities from the new set, this command is applied:\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-0J6E_t3294Y\/TrnJOjU0vWI\/AAAAAAAAAHE\/ZmI20g9iOUY\/s1600\/filter.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/1.bp.blogspot.com\/-0J6E_t3294Y\/TrnJOjU0vWI\/AAAAAAAAAHE\/ZmI20g9iOUY\/s1600\/filter.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EA quick plot of the filtered data and unfiltered data will show if weekends were properly filtered out.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/1.bp.blogspot.com\/-0k2SRSgARZE\/TrnMPJbAv8I\/AAAAAAAAAHM\/LpXYfvD7oZs\/s1600\/weekends.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/1.bp.blogspot.com\/-0k2SRSgARZE\/TrnMPJbAv8I\/AAAAAAAAAHM\/LpXYfvD7oZs\/s1600\/weekends.png\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EUnfiltered for weekends data\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cbr \/\u003E\u003Ctable align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto; text-align: center;\"\u003E\u003Ctbody\u003E\u003Ctr\u003E\u003Ctd style=\"text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-NFotI3FX3J8\/TrnMVXKkFWI\/AAAAAAAAAHU\/MLE3rdCx4vY\/s1600\/noweekends.png\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-NFotI3FX3J8\/TrnMVXKkFWI\/AAAAAAAAAHU\/MLE3rdCx4vY\/s1600\/noweekends.png\" \/\u003E\u003C\/a\u003E\u003C\/td\u003E\u003C\/tr\u003E\u003Ctr\u003E\u003Ctd class=\"tr-caption\" style=\"text-align: center;\"\u003EFiltered for weekends data\u003C\/td\u003E\u003C\/tr\u003E\u003C\/tbody\u003E\u003C\/table\u003E\u003Cdiv\u003E\u003Cbr \/\u003EThere have been a few papers that prove query volumes can be used to predict trading volumes. \u0026nbsp;But anyone can clearly see this is not the case with Apple. \u0026nbsp;This observation can be quantified with a direct Granger Causality test. \u0026nbsp;I have provided the code necessary to perform the test, but I leave it to the reader to test the lag sensitivity and establish the causal relation or feedback. \u0026nbsp;I will explain the code used for the Granger Causality test.\u003Cbr \/\u003E\u003Cbr \/\u003ETo create a new set in the form of {ylag1, ylag2, ylag3, xlag1, xlag2, xlag3, y} with a max lag of 3:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-7Qb73g9Hkgg\/TrnqoWFe_xI\/AAAAAAAAAHc\/PKCYZtgvVsc\/s1600\/lag.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" height=\"38\" src=\"http:\/\/2.bp.blogspot.com\/-7Qb73g9Hkgg\/TrnqoWFe_xI\/AAAAAAAAAHc\/PKCYZtgvVsc\/s640\/lag.png\" width=\"640\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003EThen to perform the OLS regression and sum the squared residuals:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/2.bp.blogspot.com\/-dweA3m809GI\/TrnrE790phI\/AAAAAAAAAHk\/sYRr26TAHVk\/s1600\/fit.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/2.bp.blogspot.com\/-dweA3m809GI\/TrnrE790phI\/AAAAAAAAAHk\/sYRr26TAHVk\/s1600\/fit.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003ETo compute the test statistic:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/4.bp.blogspot.com\/-FqVqRMVt46c\/TrnrTlut_DI\/AAAAAAAAAHs\/VzuNP7AlyX4\/s1600\/teststat.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/4.bp.blogspot.com\/-FqVqRMVt46c\/TrnrTlut_DI\/AAAAAAAAAHs\/VzuNP7AlyX4\/s1600\/teststat.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003EAnd since xlag = ylag, we only need to compute one f-test to get the corresponding p-value by:\u003Cbr \/\u003E\u003Cbr \/\u003E\u003Cdiv class=\"separator\" style=\"clear: both; text-align: center;\"\u003E\u003Ca href=\"http:\/\/3.bp.blogspot.com\/-_IJP4GW5K1E\/TrnsFWALSLI\/AAAAAAAAAH0\/FMqWFII4jik\/s1600\/ftest.png\" imageanchor=\"1\" style=\"clear: left; float: left; margin-bottom: 1em; margin-right: 1em;\"\u003E\u003Cimg border=\"0\" src=\"http:\/\/3.bp.blogspot.com\/-_IJP4GW5K1E\/TrnsFWALSLI\/AAAAAAAAAH0\/FMqWFII4jik\/s1600\/ftest.png\" \/\u003E\u003C\/a\u003E\u003C\/div\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003EThe Mathematica notebook for this post can be downloaded\u0026nbsp;\u003Ca href=\"http:\/\/bit.ly\/Mwg7iq\" target=\"_blank\"\u003Ehere\u003C\/a\u003E.\u003C\/div\u003E\u003Cdiv\u003E\u003Cbr \/\u003E\u003C\/div\u003E"},"link":[{"rel":"replies","type":"application/atom+xml","href":"http:\/\/nighttimecuriosities.blogspot.com\/feeds\/4497388060004690940\/comments\/default","title":"Post Comments"},{"rel":"replies","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/google-insight-data-to-predict-stock.html#comment-form","title":"0 Comments"},{"rel":"edit","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/4497388060004690940"},{"rel":"self","type":"application/atom+xml","href":"http:\/\/www.blogger.com\/feeds\/6855656749413817524\/posts\/default\/4497388060004690940"},{"rel":"alternate","type":"text/html","href":"http:\/\/nighttimecuriosities.blogspot.com\/2011\/11\/google-insight-data-to-predict-stock.html","title":"Google Insight Data to Predict Stock Market Trends, Part 1"}],"author":[{"name":{"$t":"Miguel Wong"},"uri":{"$t":"http:\/\/www.blogger.com\/profile\/00102716660812077803"},"email":{"$t":"noreply@blogger.com"},"gd$image":{"rel":"http://schemas.google.com/g/2005#thumbnail","width":"24","height":"32","src":"http:\/\/2.bp.blogspot.com\/-xykS6FaVuZE\/T9DGem_FISI\/AAAAAAAAAQI\/n1CSVlFLf40\/s220\/2012-05-20%2B14.22.20.jpg"}}],"media$thumbnail":{"xmlns$media":"http://search.yahoo.com/mrss/","url":"http:\/\/1.bp.blogspot.com\/-n-PbQnpMEzU\/Trm7OXHjefI\/AAAAAAAAAGc\/CYIkOhhdQBE\/s72-c\/path.png","height":"72","width":"72"},"thr$total":{"$t":"0"}}]}});