{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yPGVbbdyimbK"
   },
   "source": [
    "Run the code in the first two blocks to get the data, this project uses wikipedia data of cities\n",
    "\n",
    "Run the code in block three if you stored your OpenAI API key in a `.env` file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m pip install milvus llama-index python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ScX4QJ70imbN"
   },
   "outputs": [],
   "source": [
    "wiki_titles = [\"Toronto\", \"Seattle\", \"San Francisco\", \"Chicago\", \"Boston\", \"Washington, D.C.\", \"Cambridge, Massachusetts\", \"Houston\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vy1ffcjximbO"
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import requests\n",
    "for title in wiki_titles:\n",
    "     response = requests.get(\n",
    "         'https://en.wikipedia.org/w/api.php',\n",
    "         params={\n",
    "             'action': 'query',\n",
    "             'format': 'json',\n",
    "             'titles': title,\n",
    "             'prop': 'extracts',\n",
    "             # 'exintro': True,\n",
    "             'explaintext': True,\n",
    "         }\n",
    "     ).json()\n",
    "     page = next(iter(response['query']['pages'].values()))\n",
    "     wiki_text = page['extract']\n",
    "     data_path = Path('data')\n",
    "     if not data_path.exists():\n",
    "         Path.mkdir(data_path)\n",
    "     with open(data_path / f\"{title}.txt\", 'w') as fp:\n",
    "         fp.write(wiki_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "f2rsOug5imbP"
   },
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "load_dotenv()\n",
    "open_api_key = os.getenv(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oFeauEHaimbP"
   },
   "outputs": [],
   "source": [
    "from llama_index.llms import OpenAI\n",
    "from llama_index.query_engine import CitationQueryEngine\n",
    "from llama_index import (\n",
    "    VectorStoreIndex,\n",
    "    SimpleDirectoryReader,\n",
    "    StorageContext,\n",
    "    load_index_from_storage,\n",
    "    ServiceContext,\n",
    ")\n",
    "from llama_index.vector_stores import MilvusVectorStore\n",
    "\n",
    "from milvus import default_server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "N6U85o1RimbP",
    "outputId": "bdd78642-3b0a-4306-b290-3b06d6517a9b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    __  _________ _   ____  ______\n",
      "   /  |/  /  _/ /| | / / / / / __/\n",
      "  / /|_/ // // /_| |/ / /_/ /\\ \\\n",
      " /_/  /_/___/____/___/\\____/___/ {Lite}\n",
      "\n",
      " Welcome to use Milvus!\n",
      "\n",
      " Version:   v2.2.11-lite\n",
      " Process:   74039\n",
      " Started:   2023-07-21 10:11:36\n",
      " Config:    /Users/yujiantang/.milvus.io/milvus-server/2.2.11/configs/milvus.yaml\n",
      " Logs:      /Users/yujiantang/.milvus.io/milvus-server/2.2.11/logs\n",
      "\n",
      " Ctrl+C to exit ...\n"
     ]
    }
   ],
   "source": [
    "default_server.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Oil8PlU3imbQ",
    "outputId": "9ea1995b-fd6a-488a-90a7-5c5ad0afc973"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[93m[__internal_register] retry:4, cost: 0.27s, reason: <_InactiveRpcError: StatusCode.UNAVAILABLE, internal: Milvus Proxy is not ready yet. please wait>\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "vector_store = MilvusVectorStore(\n",
    "    collection_name=\"citations\",\n",
    "    host=\"127.0.0.1\",\n",
    "    port=default_server.listen_port\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LQdhatVximbR"
   },
   "outputs": [],
   "source": [
    "service_context = ServiceContext.from_defaults(\n",
    "    llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0)\n",
    ")\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ygyI_LWximbR"
   },
   "outputs": [],
   "source": [
    "documents = SimpleDirectoryReader(\"./data/\").load_data()\n",
    "index = VectorStoreIndex.from_documents(documents, service_context=service_context, storage_context=storage_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vZY9fyYiimbR"
   },
   "outputs": [],
   "source": [
    "query_engine = CitationQueryEngine.from_args(\n",
    "    index,\n",
    "    similarity_top_k=3,\n",
    "    # here we can control how granular citation sources are, the default is 512\n",
    "    citation_chunk_size=512,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ghZH6kGGimbS"
   },
   "outputs": [],
   "source": [
    "response = query_engine.query(\"Does Seattle or Houston have a bigger airport?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j38bKPOMimbS",
    "outputId": "1a4cac3f-789a-4446-9bda-c48de98b71b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Houston has a bigger airport than Seattle. George Bush Intercontinental Airport (IAH) in Houston is the eighth busiest commercial airport in the United States and the forty-third busiest globally [1]. In 2016, it served 40 million passengers, including 10 million international travelers [1]. On the other hand, Seattle-Tacoma International Airport (Sea-Tac) in Seattle is not mentioned in the provided sources.\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DuI8Um0eimbS",
    "outputId": "af30aeea-7d84-4bb7-89dd-e697e827cc46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source 1:\n",
      "Field Joint Reserve Base. The Federal Aviation Administration and the state of Texas selected the Houston Airport System as \"Airport of the Year\" in 2005, largely due to the implementation of a $3.1 billion airport improvement program for both major airports in Houston.George Bush Intercontinental Airport (IAH), 23 miles (37 km) north of Downtown Houston between Interstates 45 and 69, is the eighth busiest commercial airport in the United States (by total passengers and aircraft movements) and forty-third busiest globally. The five-terminal, five-runway, 11,000-acre (4,500-hectare) airport served 40 million passengers in 2016, including 10 million international travelers. In 2006, the United States Department of Transportation named IAH the fastest-growing of the top ten airports in the United States. The Houston Air Route Traffic Control Center is at Bush Intercontinental.\n",
      "Houston was the headquarters of Continental Airlines until its 2010 merger with United Airlines with headquarters in Chicago; regulatory approval for the merger was granted in October of that year. Bush Intercontinental is currently United Airlines' second largest hub, behind O'Hare International Airport. United Airlines' share of the Houston Airport System's commercial aviation market was nearly 60% in 2017 with 16 million enplaned passengers. In early 2007, Bush Intercontinental Airport was named a model \"port of entry\" for international travelers by U.S. Customs and Border Protection.William P. Hobby Airport (HOU), known as Houston International Airport until 1967, operates primarily short- to medium-haul domestic and international flights to 60 destinations. The four-runway, 1,304-acre (528-hectare) facility is approximately 7 miles (11 km) southeast of Downtown Houston. In 2015, Southwest Airlines launched service from a new international terminal at Hobby to several destinations in Mexico, Central America, and the Caribbean. These were the first international flights flown from Hobby since the opening of Bush Intercontinental in 1969. Houston's aviation history is showcased in the 1940 Air Terminal Museum in the old terminal building on the west side of the airport. In 2009, Hobby Airport was recognized with two awards for being one of the top five performing airports globally and for customer service by Airports Council International. In 2022 Hobby Airport was certified as the first 5-Star Airport in North America by Skytrax.\n",
      "Source 2:\n",
      "It became the first Airport in North America to do so and just the 16th airport worldwide to receive the accomplishment.Houston's third municipal airport is Ellington Airport, used by the military, government (including NASA) and general aviation sectors.== Notable people ==\n",
      "\n",
      "== International relations ==\n",
      "The Mayor's Office of Trade and International Affairs (MOTIA) is the city's liaison to Houston's sister cities and to the national governing organization, Sister Cities International. Through their official city-to-city relationships, these volunteer associations promote people-to-people diplomacy and encourage citizens to develop mutual trust and understanding through commercial, cultural, educational, and humanitarian exchanges.\n",
      "\n",
      "== See also ==\n",
      "List of people from Houston\n",
      "List of U.S. cities with large Hispanic populations\n",
      "USS Houston, 4 ships== Notes ==\n",
      "\n",
      "== References ==\n",
      "\n",
      "== Further reading ==\n",
      "\n",
      "== External links ==\n",
      "\n",
      "Official website \n",
      "Greater Houston Convention & Visitors Bureau\n",
      "Greater Houston Partnership (GHP) Houston Chamber\n",
      "Greater Houston Transportation and Emergency Management Center\n",
      "Houston at Curlie\n",
      "Source 3:\n",
      "in the world to have a heart attack\". Three of Seattle's largest medical centers are located on First Hill. Harborview Medical Center, the public county hospital, is the only Level I trauma hospital in a region that includes Washington, Alaska, Montana, and Idaho. Virginia Mason Medical Center and Swedish Medical Center's two largest campuses are also located in this part of Seattle, including the Virginia Mason Hospital. This concentration of hospitals resulted in the neighborhood's nickname \"Pill Hill\". Located in the Laurelhurst neighborhood, Seattle Children's, formerly Children's Hospital and Regional Medical Center, is the pediatric referral center for Washington, Alaska, Montana, and Idaho. The Fred Hutchinson Cancer Research Center has a campus in the Eastlake neighborhood. The University District is home to the University of Washington Medical Center which, along with Harborview, is operated by the University of Washington. Seattle is also served by a Veterans Affairs hospital on Beacon Hill, a third campus of Swedish in Ballard, and UW Medical Center - Northwest near Northgate Station.=== Transportation ===\n",
      "\n",
      "The first streetcars appeared in 1889 and were instrumental in the creation of a relatively well-defined downtown and strong neighborhoods at the end of their lines. The advent of the automobile began the dismantling of rail in Seattle. Tacoma–Seattle railway service ended in 1929 and the Everett–Seattle service came to an end in 1939, replaced by automobiles running on the recently developed highway system. Rails on city streets were paved over or removed, and the opening of the Seattle trolleybus system brought the end of streetcars in Seattle in 1941. This left an extensive network of privately owned buses (later public) as the only mass transit within the city and throughout the region.King County Metro provides frequent stop bus service within the city and surrounding county, as well as the South Lake Union Streetcar line and the First Hill Streetcar line. Seattle is one of the few cities in North America whose bus fleet includes electric trolleybuses. Sound Transit provides an express bus service within the metropolitan area, two Sounder commuter rail lines between the suburbs and downtown, and its 1 Line light rail line between the University of Washington and Angle Lake. Washington State Ferries, which manages the largest network of ferries in the United States and third largest in the world, connects Seattle to Bainbridge and Vashon Islands in Puget Sound and to Bremerton and Southworth on the Kitsap Peninsula.\n",
      "Source 4:\n",
      "King Street Station in Pioneer Square serves Amtrak intercity trains and Sounder commuter trains, and is located adjacent to the International District/Chinatown light rail station.According to the 2007 American Community Survey, 18.6% of Seattle residents used one of the three public transit systems that serve the city, giving it the highest transit ridership of all major cities without heavy or light rail prior to the completion of Sound Transit's 1 Line. The city has also been described by Bert Sperling as the fourth most walkable U.S. city and by Walk Score as the sixth most walkable of the fifty largest U.S. cities.Seattle–Tacoma International Airport, locally known as Sea-Tac Airport and located just south in the neighboring city of SeaTac, is operated by the Port of Seattle and provides commercial air service to destinations throughout the world. Closer to downtown, Boeing Field is used for general aviation, cargo flights, and testing/delivery of Boeing airliners. A secondary passenger airport, Paine Field, opened in 2019 and is located in Everett, 25 miles (40 km) north of Seattle. It is predominantly used by Boeing and their large assembly plant located nearby.The main mode of transportation, however, is the street system, which is laid out in a cardinal directions grid pattern, except in the central business district where early city leaders Arthur Denny and Carson Boren insisted on orienting the plats relative to the shoreline rather than to true North. Only two roads, Interstate 5 and State Route 99 (both limited-access highways) run uninterrupted through the city from north to south. From 1953 to 2019, State Route 99 ran through downtown Seattle on the Alaskan Way Viaduct,\n",
      "Source 5:\n",
      "permanent attraction on the city's waterfront, at Pier 57, next to Downtown Seattle. The city also has many community centers for recreation, including Rainier Beach, Van Asselt, Rainier, and Jefferson south of the Ship Canal and Green Lake, Laurelhurst, Loyal Heights north of the Canal, and Meadowbrook.Woodland Park Zoo opened as a private menagerie in 1889 but was sold to the city in 1899. The Seattle Aquarium has been open on the downtown waterfront since 1977 (undergoing a renovation in 2006). The Seattle Underground Tour is an exhibit of places that existed before the Great Fire.Since the middle 1990s, Seattle has experienced significant growth in the cruise industry, especially as a departure point for Alaska cruises. In 2008, a record total of 886,039 cruise passengers passed through the city, surpassing the number for Vancouver, BC, the other major departure point for Alaska cruises.== Religion ==\n",
      "According to a 2014 study by the Pew Research Center, the largest religious groupings are Christians (52%), followed by those of no religion (37%), Hindus (2%), Buddhists (2%), Jews (1%), Muslims (1%) and a variety of other religions have smaller followings. According to the same study by the Pew Research Center, about 34% of Seattleites are Protestant, and 15% are Roman Catholic. Meanwhile, 6% of the residents in Seattle call themselves agnostics, while 10% call themselves atheists.\n",
      "\n",
      "== Sports ==\n",
      "\n",
      "Seattle has four major men's professional sports teams: the National Football League (NFL)'s Seattle Seahawks, Major League Baseball (MLB)'s Seattle Mariners, the National Hockey League (NHL)'s Seattle Kraken, and Major League Soccer (MLS)'s Seattle Sounders FC. Other professional sports teams include the Women's National Basketball Association (WNBA)'s Seattle Storm, the National Women's Soccer League's OL Reign; and Major League Rugby (MLR)'s Seattle Seawolves.\n",
      "Seattle's professional sports history began at the start of the 20th century with the PCHA's Seattle Metropolitans, which in 1917 became the first American hockey team to win the Stanley Cup.\n",
      "In 1969, Seattle was awarded a Major League Baseball franchise, the Seattle Pilots. Based at Sick's Stadium in Mount Baker, home to Seattle's former minor-league teams,\n",
      "Source 6:\n",
      "the Pilots played in Seattle for one season before relocating to Milwaukee and becoming the Milwaukee Brewers. The city, alongside the county and state governments, sued the league and was offered a second expansion team, the Seattle Mariners, as settlement.\n",
      "The Mariners began play in 1977 at the multi-purpose Kingdome, where the team struggled for most of its time. Relative success in the mid-to-late 1990s saved the team from being relocated and allowed them to move to a purpose-built baseball stadium, T-Mobile Park (formerly Safeco Field), in 1999. The Mariners have never reached a World Series and only appeared in the MLB playoffs five times, mostly between 1995 and 2001, but had Hall of Fame players and candidates like Ken Griffey Jr. Randy Johnson, Ichiro Suzuki, and Alex Rodriguez. The team tied the all-time MLB single regular season wins record in 2001 with 116 wins. From 2001 to 2022, the Mariners failed to qualify for the playoffs—the longest active postseason drought in major North American sports, at 20 seasons.The Seattle Seahawks entered the National Football League in 1976 as an expansion team and have advanced to the Super Bowl three times: 2005, 2013 and 2014. The team played in the Kingdome until it was imploded in 2000 and moved into Qwest Field (now Lumen Field) at the same site in 2003. The Seahawks lost Super Bowl XL in 2005 to the Pittsburgh Steelers in Detroit, but won Super Bowl XLVIII in 2013 by defeating the Denver Broncos 43–8 at MetLife Stadium. The team advanced to the Super Bowl the following year, but lost to the New England Patriots in Super Bowl XLIX on a last-minute play. Seahawks fans have set stadium noise records on several occasions and are collectively known as the \"12th\n"
     ]
    }
   ],
   "source": [
    "for source in response.source_nodes:\n",
    "    print(source.node.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zRCwkGfWimbS"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}