{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"u0t1UALWvzbq"},"source":["# 1st LAB: Working with Documents"]},{"cell_type":"markdown","source":["You can mount a google drive to connect you drive storage or start this notebook and temporarily add the file \n","\n","> **BEWARE: if you use the temporary storage the file is be removed after the notebook is closed!**\n","\n","\n"],"metadata":{"id":"8NeNlLCNIZCm"}},{"cell_type":"code","metadata":{"id":"9jQRNFjwTuFb","colab":{"base_uri":"https://localhost:8080/"},"outputId":"c2a29355-5faa-436d-cfb0-f51d0eb55a49","executionInfo":{"status":"ok","timestamp":1668005874792,"user_tz":-60,"elapsed":2379,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["from google.colab import drive\n","drive.mount(\"/content/gdrive\")\n","\n","from collections import defaultdict\n","from pathlib import Path\n","import pandas as pd\n","pd.set_option('display.max_colwidth', -1)"],"execution_count":46,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n"," import sys\n"]}]},{"cell_type":"code","source":["!ls gdrive/MyDrive/LAB-1_IR_2022-2023"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nhEnUSx-JTj_","executionInfo":{"status":"ok","timestamp":1668005881358,"user_tz":-60,"elapsed":717,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"8aa0439f-86f6-476a-a653-6f0ccc171e06"},"execution_count":47,"outputs":[{"output_type":"stream","name":"stdout","text":[" Document_Representation.ipynb\t spam.csv\n","'IR LAB 1 - Slides.pdf'\t\t Text_Preprocessing.ipynb\n"]}]},{"cell_type":"markdown","metadata":{"id":"IW14-Hn8JvQj"},"source":["# Importing the data"]},{"cell_type":"code","source":["import pandas as pd\n","\n","## Reading the data from a csv from\n","\n","# base_path = ''\n","base_path = 'gdrive/MyDrive/LAB-1_IR_2022-2023/' # use the path to your gdrive folder\n","\n","data_spam = pd.read_csv(f\"{base_path}spam.csv\", encoding=\"ISO-8859-1\")\n","display(data_spam.head())\n","print(data_spam.shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":224},"id":"OFNk2RVJKXCB","executionInfo":{"status":"ok","timestamp":1668006557497,"user_tz":-60,"elapsed":533,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"4c23a3af-c488-475d-f924-44684734c7f7"},"execution_count":48,"outputs":[{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","2 spam \n","3 ham \n","4 ham \n","\n"," v2 \\\n","0 Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat... \n","1 Ok lar... Joking wif u oni... \n","2 Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's \n","3 U dun say so early hor... U c already then say... \n","4 Nah I don't think he goes to usf, he lives around here though \n","\n"," Unnamed: 2 Unnamed: 3 Unnamed: 4 \n","0 NaN NaN NaN \n","1 NaN NaN NaN \n","2 NaN NaN NaN \n","3 NaN NaN NaN \n","4 NaN NaN NaN "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2Unnamed: 2Unnamed: 3Unnamed: 4
0hamGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...NaNNaNNaN
1hamOk lar... Joking wif u oni...NaNNaNNaN
2spamFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18'sNaNNaNNaN
3hamU dun say so early hor... U c already then say...NaNNaNNaN
4hamNah I don't think he goes to usf, he lives around here thoughNaNNaNNaN
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["(5572, 5)\n"]}]},{"cell_type":"code","source":["display(data_spam[data_spam['Unnamed: 2'].notna()].head())\n","print(data_spam[data_spam['Unnamed: 2'].notna()].shape)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":355},"id":"O7mywTn4KakS","executionInfo":{"status":"ok","timestamp":1667998548548,"user_tz":-60,"elapsed":302,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"8ef76e20-168b-4ded-b48b-0125e5165698"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","95 spam \n","281 ham \n","444 ham \n","671 spam \n","710 ham \n","\n"," v2 \\\n","95 Your free ringtone is waiting to be collected. Simply text the password \\MIX\\\" to 85069 to verify. Get Usher and Britney. FML \n","281 \\Wen u miss someone \n","444 \\HEY HEY WERETHE MONKEESPEOPLE SAY WE MONKEYAROUND! HOWDY GORGEOUS \n","671 SMS. ac sun0819 posts HELLO:\\You seem cool \n","710 Height of Confidence: All the Aeronautics professors wer calld & they wer askd 2 sit in an aeroplane. Aftr they sat they wer told dat the plane ws made by their students. Dey all hurried out of d plane.. Bt only 1 didnt move... He said:\\if it is made by my students \n","\n"," Unnamed: 2 \\\n","95 PO Box 5249 \n","281 the person is definitely special for u..... But if the person is so special \n","444 HOWU DOIN? FOUNDURSELF A JOBYET SAUSAGE?LOVE JEN XXX\\\"\" \n","671 wanted to say hi. HI!!!\\\" Stop? Send STOP to 62468\" \n","710 this wont even start........ Datz confidence..\" \n","\n"," Unnamed: 3 Unnamed: 4 \n","95 MK17 92H. 450Ppw 16\" NaN \n","281 why to miss them just Keep-in-touch\\\" gdeve..\" \n","444 NaN NaN \n","671 NaN NaN \n","710 NaN NaN "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2Unnamed: 2Unnamed: 3Unnamed: 4
95spamYour free ringtone is waiting to be collected. Simply text the password \\MIX\\\" to 85069 to verify. Get Usher and Britney. FMLPO Box 5249MK17 92H. 450Ppw 16\"NaN
281ham\\Wen u miss someonethe person is definitely special for u..... But if the person is so specialwhy to miss themjust Keep-in-touch\\\" gdeve..\"
444ham\\HEY HEY WERETHE MONKEESPEOPLE SAY WE MONKEYAROUND! HOWDY GORGEOUSHOWU DOIN? FOUNDURSELF A JOBYET SAUSAGE?LOVE JEN XXX\\\"\"NaNNaN
671spamSMS. ac sun0819 posts HELLO:\\You seem coolwanted to say hi. HI!!!\\\" Stop? Send STOP to 62468\"NaNNaN
710hamHeight of Confidence: All the Aeronautics professors wer calld & they wer askd 2 sit in an aeroplane. Aftr they sat they wer told dat the plane ws made by their students. Dey all hurried out of d plane.. Bt only 1 didnt move... He said:\\if it is made by my studentsthis wont even start........ Datz confidence..\"NaNNaN
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["(50, 5)\n"]}]},{"cell_type":"code","source":["data_spam = data_spam[['v1','v2']]\n","display(data_spam.head())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":285},"id":"INxNYNw4KHZp","executionInfo":{"status":"ok","timestamp":1668006699126,"user_tz":-60,"elapsed":232,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"08aced06-2af0-4916-ea28-c0bec653edf5"},"execution_count":49,"outputs":[{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","2 spam \n","3 ham \n","4 ham \n","\n"," v2 \n","0 Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat... \n","1 Ok lar... Joking wif u oni... \n","2 Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's \n","3 U dun say so early hor... U c already then say... \n","4 Nah I don't think he goes to usf, he lives around here though "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2
0hamGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...
1hamOk lar... Joking wif u oni...
2spamFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's
3hamU dun say so early hor... U c already then say...
4hamNah I don't think he goes to usf, he lives around here though
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}}]},{"cell_type":"code","metadata":{"id":"ertxZLqZKQH8","colab":{"base_uri":"https://localhost:8080/"},"outputId":"531edba5-da7a-4f39-961c-55ebcdf1304a","executionInfo":{"status":"ok","timestamp":1668006704244,"user_tz":-60,"elapsed":229,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["import nltk\n","from nltk.corpus import movie_reviews\n","nltk.download('movie_reviews') ###< run once\n","nltk.download('punkt') ###< run once"],"execution_count":50,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package movie_reviews to /root/nltk_data...\n","[nltk_data] Package movie_reviews is already up-to-date!\n","[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data] Package punkt is already up-to-date!\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":50}]},{"cell_type":"markdown","source":["Data for google colab gets saved here:\n","\n","`/root/nltk_data/corpora/movie_reviews`"],"metadata":{"id":"FRKcGBNiPkCp"}},{"cell_type":"code","source":["!ls /root/nltk_data/corpora/movie_reviews"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FCEVOnLUPr49","executionInfo":{"status":"ok","timestamp":1668006744644,"user_tz":-60,"elapsed":232,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"bb4b6fa9-7a45-4588-aae0-c8e8aba2d503"},"execution_count":51,"outputs":[{"output_type":"stream","name":"stdout","text":["neg pos README\n"]}]},{"cell_type":"markdown","source":["What is this dataset all about?"],"metadata":{"id":"ZNFgRfO3P8Rj"}},{"cell_type":"code","source":["print(movie_reviews.readme()[:500])\n","print('...')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"v_suXQmrPxSU","executionInfo":{"status":"ok","timestamp":1668006760486,"user_tz":-60,"elapsed":247,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"0935b531-54f2-45be-ffd9-88a3737c6f7f"},"execution_count":52,"outputs":[{"output_type":"stream","name":"stdout","text":["Sentiment Polarity Dataset Version 2.0\n","Bo Pang and Lillian Lee\n","\n","http://www.cs.cornell.edu/people/pabo/movie-review-data/\n","\n","Distributed with NLTK with permission from the authors.\n","\n","=======\n","\n","Introduction\n","\n","This README v2.0 (June, 2004) for the v2.0 polarity dataset comes from\n","the URL http://www.cs.cornell.edu/people/pabo/movie-review-data .\n","\n","=======\n","\n","What's New -- June, 2004\n","\n","This dataset represents an enhancement of the review corpus v1.0\n","described in README v1.1: it contains more reviews, and labe\n","...\n"]}]},{"cell_type":"code","source":["raw = movie_reviews.raw()\n","print(f\"RAW representation of movies reviews dataset, length {len(movie_reviews.words())}\\n\")\n","print(raw[:500])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PfDOFaS6K3lS","executionInfo":{"status":"ok","timestamp":1668006833695,"user_tz":-60,"elapsed":1589,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"99301063-a4ed-46d1-9e2f-15f4a456780c"},"execution_count":53,"outputs":[{"output_type":"stream","name":"stdout","text":["RAW representation of movies reviews dataset, length 1583820\n","\n","plot : two teen couples go to a church party , drink and then drive . \n","they get into an accident . \n","one of the guys dies , but his girlfriend continues to see him in her life , and has nightmares . \n","what's the deal ? \n","watch the movie and \" sorta \" find out . . . \n","critique : a mind-fuck movie for the teen generation that touches on a very cool idea , but presents it in a very bad package . \n","which is what makes this review an even harder one to write , since i generally applaud films which attempt\n"]}]},{"cell_type":"markdown","metadata":{"id":"kwbEXfo8KQXr"},"source":["# Remove whitespaces"]},{"cell_type":"code","source":["print('Sample Text:\\n')\n","text = \"DEAR FRIEND,\\n\\nI AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) WHO WAS ASSASSINATED BY HIS BODY GUARD ON 16TH JAN. 2001.\\n\\n\\nTHE INCIDENT OCCURRED IN OUR PRESENCE WHILE WE WERE HOLDING MEETING WITH HIS EXCELLENCY OVER THE FINANCIAL RETURNS FROM THE DIAMOND SALES IN THE AREAS CONTROLLED BY (D.R.C.) DEMOCRATIC REPUBLIC OF CONGO FORCES AND THEIR FOREIGN ALLIES ANGOLA AND ZIMBABWE, HAVING RECEIVED THE PREVIOUS DAY (USD$100M) ONE HUNDRED MILLION UNITED STATES DOLLARS, CASH IN THREE DIPLOMATIC BOXES ROUTED THROUGH ZIMBABWE.\\n\\nMY PURPOSE OF WRITING YOU THIS LETTER IS TO SOLICIT FOR YOUR ASSISTANCE AS TO BE A COVER TO THE FUND AND ALSO COLLABORATION IN MOVING THE SAID FUND INTO YOUR BANK ACCOUNT THE SUM OF (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY, WHICH I DEPOSITED WITH A SECURITY COMPANY IN GHANA, IN A DIPLOMATIC BOX AS GOLDS WORTH (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY FOR SAFE KEEPING IN A SECURITY VAULT FOR ANY FURTHER INVESTMENT PERHAPS IN YOUR COUNTRY. \\n\\nYOU WERE INTRODUCED TO ME BY A RELIABLE FRIEND OF MINE WHO IS A TRAVELLER,AND ALSO A MEMBER OF CHAMBER OF COMMERCE AS A RELIABLE AND TRUSTWORTHY PERSON WHOM I CAN RELY ON AS FOREIGN PARTNER, EVEN THOUGH THE NATURE OF THE TRANSACTION WAS NOT REVEALED TO HIM FOR SECURITY REASONS.\\n\\n\\nTHE (USD$25M) WAS PART OF A PROCEEDS FROM DIAMOND TRADE MEANT FOR THE LATE PRESIDENT LAURENT KABILA WHICH WAS DELIVERED THROUGH ZIMBABWE IN DIPLOMATIC BOXES. THE BOXES WERE KEPT UNDER MY CUSTODY BEFORE THE SAD EVENT THAT TOOK THE LIFE OF (MR. PRESIDENT).THE CONFUSION THAT ENSUED AFTER THE ASSASSINATION AND THE SPORADIC SHOOTING AMONG THE FACTIONS, I HAVE TO RUN AWAY FROM THE COUNTRY FOR MY DEAR LIFE AS I AM NOT A SOLDIER BUT A CIVIL SERVANT I CROSSED RIVER CONGO TO OTHER SIDE OF CONGO LIBREVILLE FROM THERE I MOVED TO THE THIRD COUNTRY GHANA WHERE I AM PRESENTLY TAKING REFUGE. \\n\\nAS A MATTER OF FACT, WHAT I URGENTLY NEEDED FROM YOU IS YOUR ASSISTANCE IN MOVING THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTRY FOR INVESTMENT WITHOUT RAISING EYEBROW. FOR YOUR ASSISTANCE I WILL GIVE YOU 20% OF THE TOTAL SUM AS YOUR OWN SHARE WHEN THE MONEY GETS TO YOUR ACCOUNT, WHILE 75% WILL BE FOR ME, OF WHICH WITH YOUR KIND ADVICE I HOPE TO INVEST IN PROFITABLE VENTURE IN YOUR COUNTRY IN OTHER TO SETTLE DOWN FOR MEANINGFUL LIFE, AS I AM TIRED OF LIVING IN A WAR ENVIRONMENT. \\n\\nTHE REMAINING 5% WILL BE USED TO OFFSET ANY COST INCURRED IN THE CAUSE OF MOVING THE MONEY TO YOUR ACCOUNT. IF THE PROPOSAL IS ACCEPTABLE TO YOU PLEASE CONTACT ME IMMEDIATELY THROUGH THE ABOVE TELEPHONE AND E-MAIL, TO ENABLE ME ARRANGE FACE TO FACE MEETING WITH YOU IN GHANA FOR THE CLEARANCE OF THE FUNDS BEFORE TRANSFRING IT TO YOUR BANK ACCOUNT AS SEEING IS BELIEVING. \\n\\nFINALLY, IT IS IMPORTANT ALSO THAT I LET YOU UNDERSTAND THAT THERE IS NO RISK INVOLVED WHATSOEVER AS THE MONEY HAD NO RECORD IN KINSHASA FOR IT WAS MEANT FOR THE PERSONAL USE OF (MR. PRESIDEND ) BEFORE THE NEFARIOUS INCIDENT OCCURRED, AND ALSO I HAVE ALL THE NECESSARY DOCUMENTS AS REGARDS TO THE FUNDS INCLUDING THE (CERTIFICATE OF DEPOSIT), AS I AM THE DEPOSITOR OF THE CONSIGNMENT.\\n\\n\\nLOOKING FORWARD TO YOUR URGENT RESPONSE.\\n\\nYOUR SINCERELY,\\n\\n\\t\\t\\tMR. JAMES NGOLA. \" \n","print(text)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RgOlJFQ_QJf6","executionInfo":{"status":"ok","timestamp":1668006880244,"user_tz":-60,"elapsed":274,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"389a673c-9797-4951-de85-7e719af662f5"},"execution_count":54,"outputs":[{"output_type":"stream","name":"stdout","text":["Sample Text:\n","\n","DEAR FRIEND,\n","\n","I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) WHO WAS ASSASSINATED BY HIS BODY GUARD ON 16TH JAN. 2001.\n","\n","\n","THE INCIDENT OCCURRED IN OUR PRESENCE WHILE WE WERE HOLDING MEETING WITH HIS EXCELLENCY OVER THE FINANCIAL RETURNS FROM THE DIAMOND SALES IN THE AREAS CONTROLLED BY (D.R.C.) DEMOCRATIC REPUBLIC OF CONGO FORCES AND THEIR FOREIGN ALLIES ANGOLA AND ZIMBABWE, HAVING RECEIVED THE PREVIOUS DAY (USD$100M) ONE HUNDRED MILLION UNITED STATES DOLLARS, CASH IN THREE DIPLOMATIC BOXES ROUTED THROUGH ZIMBABWE.\n","\n","MY PURPOSE OF WRITING YOU THIS LETTER IS TO SOLICIT FOR YOUR ASSISTANCE AS TO BE A COVER TO THE FUND AND ALSO COLLABORATION IN MOVING THE SAID FUND INTO YOUR BANK ACCOUNT THE SUM OF (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY, WHICH I DEPOSITED WITH A SECURITY COMPANY IN GHANA, IN A DIPLOMATIC BOX AS GOLDS WORTH (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY FOR SAFE KEEPING IN A SECURITY VAULT FOR ANY FURTHER INVESTMENT PERHAPS IN YOUR COUNTRY. \n","\n","YOU WERE INTRODUCED TO ME BY A RELIABLE FRIEND OF MINE WHO IS A TRAVELLER,AND ALSO A MEMBER OF CHAMBER OF COMMERCE AS A RELIABLE AND TRUSTWORTHY PERSON WHOM I CAN RELY ON AS FOREIGN PARTNER, EVEN THOUGH THE NATURE OF THE TRANSACTION WAS NOT REVEALED TO HIM FOR SECURITY REASONS.\n","\n","\n","THE (USD$25M) WAS PART OF A PROCEEDS FROM DIAMOND TRADE MEANT FOR THE LATE PRESIDENT LAURENT KABILA WHICH WAS DELIVERED THROUGH ZIMBABWE IN DIPLOMATIC BOXES. THE BOXES WERE KEPT UNDER MY CUSTODY BEFORE THE SAD EVENT THAT TOOK THE LIFE OF (MR. PRESIDENT).THE CONFUSION THAT ENSUED AFTER THE ASSASSINATION AND THE SPORADIC SHOOTING AMONG THE FACTIONS, I HAVE TO RUN AWAY FROM THE COUNTRY FOR MY DEAR LIFE AS I AM NOT A SOLDIER BUT A CIVIL SERVANT I CROSSED RIVER CONGO TO OTHER SIDE OF CONGO LIBREVILLE FROM THERE I MOVED TO THE THIRD COUNTRY GHANA WHERE I AM PRESENTLY TAKING REFUGE. \n","\n","AS A MATTER OF FACT, WHAT I URGENTLY NEEDED FROM YOU IS YOUR ASSISTANCE IN MOVING THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTRY FOR INVESTMENT WITHOUT RAISING EYEBROW. FOR YOUR ASSISTANCE I WILL GIVE YOU 20% OF THE TOTAL SUM AS YOUR OWN SHARE WHEN THE MONEY GETS TO YOUR ACCOUNT, WHILE 75% WILL BE FOR ME, OF WHICH WITH YOUR KIND ADVICE I HOPE TO INVEST IN PROFITABLE VENTURE IN YOUR COUNTRY IN OTHER TO SETTLE DOWN FOR MEANINGFUL LIFE, AS I AM TIRED OF LIVING IN A WAR ENVIRONMENT. \n","\n","THE REMAINING 5% WILL BE USED TO OFFSET ANY COST INCURRED IN THE CAUSE OF MOVING THE MONEY TO YOUR ACCOUNT. IF THE PROPOSAL IS ACCEPTABLE TO YOU PLEASE CONTACT ME IMMEDIATELY THROUGH THE ABOVE TELEPHONE AND E-MAIL, TO ENABLE ME ARRANGE FACE TO FACE MEETING WITH YOU IN GHANA FOR THE CLEARANCE OF THE FUNDS BEFORE TRANSFRING IT TO YOUR BANK ACCOUNT AS SEEING IS BELIEVING. \n","\n","FINALLY, IT IS IMPORTANT ALSO THAT I LET YOU UNDERSTAND THAT THERE IS NO RISK INVOLVED WHATSOEVER AS THE MONEY HAD NO RECORD IN KINSHASA FOR IT WAS MEANT FOR THE PERSONAL USE OF (MR. PRESIDEND ) BEFORE THE NEFARIOUS INCIDENT OCCURRED, AND ALSO I HAVE ALL THE NECESSARY DOCUMENTS AS REGARDS TO THE FUNDS INCLUDING THE (CERTIFICATE OF DEPOSIT), AS I AM THE DEPOSITOR OF THE CONSIGNMENT.\n","\n","\n","LOOKING FORWARD TO YOUR URGENT RESPONSE.\n","\n","YOUR SINCERELY,\n","\n","\t\t\tMR. JAMES NGOLA. \n"]}]},{"cell_type":"code","metadata":{"id":"Xl8cKKlIKQgR","colab":{"base_uri":"https://localhost:8080/"},"outputId":"8ab02fe1-5e04-45d3-9869-ba65b310890f","executionInfo":{"status":"ok","timestamp":1668006906012,"user_tz":-60,"elapsed":412,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["print('Initial text lenght:',len(text))\n","print(\"Number of 2 newlines\",text.count('\\n\\n'))\n","print(\"Number of tabs\",text.count('\\t'))\n","print(\"Number of 2 tabs\",text.count('\\t\\t'))\n","print(\"Number of tab and newline\",text.count('\\n\\t'))"],"execution_count":55,"outputs":[{"output_type":"stream","name":"stdout","text":["Initial text lenght: 3237\n","Number of 2 newlines 11\n","Number of tabs 3\n","Number of 2 tabs 1\n","Number of tab and newline 1\n"]}]},{"cell_type":"code","source":["print('\\nManual approach:')\n","new_text = text.replace('\\\\n', ' ').replace('\\n', ' ').replace('\\t',' ').replace('\\\\', ' ').replace('\\n\\n',' ')\n","print(new_text)\n","print(\"Number of 2 newlines\",new_text.count('\\n\\n'))\n","print(\"Number of tabs\",new_text.count('\\t'))\n","print(\"Number of 2 tabs\",new_text.count('\\t\\t'))\n","print(\"Number of tab and newline\",new_text.count('\\n\\t'))\n","print('Manual text lenght',len(new_text))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XGzV3XlnQO8L","executionInfo":{"status":"ok","timestamp":1668006979414,"user_tz":-60,"elapsed":246,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"2d68add2-2308-42d3-b178-c72abb19b235"},"execution_count":56,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Manual approach:\n","DEAR FRIEND, I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) WHO WAS ASSASSINATED BY HIS BODY GUARD ON 16TH JAN. 2001. THE INCIDENT OCCURRED IN OUR PRESENCE WHILE WE WERE HOLDING MEETING WITH HIS EXCELLENCY OVER THE FINANCIAL RETURNS FROM THE DIAMOND SALES IN THE AREAS CONTROLLED BY (D.R.C.) DEMOCRATIC REPUBLIC OF CONGO FORCES AND THEIR FOREIGN ALLIES ANGOLA AND ZIMBABWE, HAVING RECEIVED THE PREVIOUS DAY (USD$100M) ONE HUNDRED MILLION UNITED STATES DOLLARS, CASH IN THREE DIPLOMATIC BOXES ROUTED THROUGH ZIMBABWE. MY PURPOSE OF WRITING YOU THIS LETTER IS TO SOLICIT FOR YOUR ASSISTANCE AS TO BE A COVER TO THE FUND AND ALSO COLLABORATION IN MOVING THE SAID FUND INTO YOUR BANK ACCOUNT THE SUM OF (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY, WHICH I DEPOSITED WITH A SECURITY COMPANY IN GHANA, IN A DIPLOMATIC BOX AS GOLDS WORTH (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY FOR SAFE KEEPING IN A SECURITY VAULT FOR ANY FURTHER INVESTMENT PERHAPS IN YOUR COUNTRY. YOU WERE INTRODUCED TO ME BY A RELIABLE FRIEND OF MINE WHO IS A TRAVELLER,AND ALSO A MEMBER OF CHAMBER OF COMMERCE AS A RELIABLE AND TRUSTWORTHY PERSON WHOM I CAN RELY ON AS FOREIGN PARTNER, EVEN THOUGH THE NATURE OF THE TRANSACTION WAS NOT REVEALED TO HIM FOR SECURITY REASONS. THE (USD$25M) WAS PART OF A PROCEEDS FROM DIAMOND TRADE MEANT FOR THE LATE PRESIDENT LAURENT KABILA WHICH WAS DELIVERED THROUGH ZIMBABWE IN DIPLOMATIC BOXES. THE BOXES WERE KEPT UNDER MY CUSTODY BEFORE THE SAD EVENT THAT TOOK THE LIFE OF (MR. PRESIDENT).THE CONFUSION THAT ENSUED AFTER THE ASSASSINATION AND THE SPORADIC SHOOTING AMONG THE FACTIONS, I HAVE TO RUN AWAY FROM THE COUNTRY FOR MY DEAR LIFE AS I AM NOT A SOLDIER BUT A CIVIL SERVANT I CROSSED RIVER CONGO TO OTHER SIDE OF CONGO LIBREVILLE FROM THERE I MOVED TO THE THIRD COUNTRY GHANA WHERE I AM PRESENTLY TAKING REFUGE. AS A MATTER OF FACT, WHAT I URGENTLY NEEDED FROM YOU IS YOUR ASSISTANCE IN MOVING THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTRY FOR INVESTMENT WITHOUT RAISING EYEBROW. FOR YOUR ASSISTANCE I WILL GIVE YOU 20% OF THE TOTAL SUM AS YOUR OWN SHARE WHEN THE MONEY GETS TO YOUR ACCOUNT, WHILE 75% WILL BE FOR ME, OF WHICH WITH YOUR KIND ADVICE I HOPE TO INVEST IN PROFITABLE VENTURE IN YOUR COUNTRY IN OTHER TO SETTLE DOWN FOR MEANINGFUL LIFE, AS I AM TIRED OF LIVING IN A WAR ENVIRONMENT. THE REMAINING 5% WILL BE USED TO OFFSET ANY COST INCURRED IN THE CAUSE OF MOVING THE MONEY TO YOUR ACCOUNT. IF THE PROPOSAL IS ACCEPTABLE TO YOU PLEASE CONTACT ME IMMEDIATELY THROUGH THE ABOVE TELEPHONE AND E-MAIL, TO ENABLE ME ARRANGE FACE TO FACE MEETING WITH YOU IN GHANA FOR THE CLEARANCE OF THE FUNDS BEFORE TRANSFRING IT TO YOUR BANK ACCOUNT AS SEEING IS BELIEVING. FINALLY, IT IS IMPORTANT ALSO THAT I LET YOU UNDERSTAND THAT THERE IS NO RISK INVOLVED WHATSOEVER AS THE MONEY HAD NO RECORD IN KINSHASA FOR IT WAS MEANT FOR THE PERSONAL USE OF (MR. PRESIDEND ) BEFORE THE NEFARIOUS INCIDENT OCCURRED, AND ALSO I HAVE ALL THE NECESSARY DOCUMENTS AS REGARDS TO THE FUNDS INCLUDING THE (CERTIFICATE OF DEPOSIT), AS I AM THE DEPOSITOR OF THE CONSIGNMENT. LOOKING FORWARD TO YOUR URGENT RESPONSE. YOUR SINCERELY, MR. JAMES NGOLA. \n","Number of 2 newlines 0\n","Number of tabs 0\n","Number of 2 tabs 0\n","Number of tab and newline 0\n","Manual text lenght 3237\n"]}]},{"cell_type":"code","source":["print('\\nRegex library:')\n","import re \n","new_text2 = re.sub(r'\\s\\s', '', text)\n","print(new_text2)\n","print(\"Number of 2 newlines\",new_text2.count('\\n\\n'))\n","print(\"Number of tabs\",new_text2.count('\\t'))\n","print(\"Number of 2 tabs\",new_text2.count('\\t\\t'))\n","print(\"Number of tab and newline\",new_text2.count('\\n\\t'))\n","print('Regex text lenght',len(new_text2))\n","\n","text = new_text2"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5bT_5kMlQRLc","executionInfo":{"status":"ok","timestamp":1668007059823,"user_tz":-60,"elapsed":228,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"e3945382-a383-4c10-a03e-18143b019fc5"},"execution_count":57,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Regex library:\n","DEAR FRIEND,I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) WHO WAS ASSASSINATED BY HIS BODY GUARD ON 16TH JAN. 2001.\n","THE INCIDENT OCCURRED IN OUR PRESENCE WHILE WE WERE HOLDING MEETING WITH HIS EXCELLENCY OVER THE FINANCIAL RETURNS FROM THE DIAMOND SALES IN THE AREAS CONTROLLED BY (D.R.C.) DEMOCRATIC REPUBLIC OF CONGO FORCES AND THEIR FOREIGN ALLIES ANGOLA AND ZIMBABWE, HAVING RECEIVED THE PREVIOUS DAY (USD$100M) ONE HUNDRED MILLION UNITED STATES DOLLARS, CASH IN THREE DIPLOMATIC BOXES ROUTED THROUGH ZIMBABWE.MY PURPOSE OF WRITING YOU THIS LETTER IS TO SOLICIT FOR YOUR ASSISTANCE AS TO BE A COVER TO THE FUND AND ALSO COLLABORATION IN MOVING THE SAID FUND INTO YOUR BANK ACCOUNT THE SUM OF (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY, WHICH I DEPOSITED WITH A SECURITY COMPANY IN GHANA, IN A DIPLOMATIC BOX AS GOLDS WORTH (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY FOR SAFE KEEPING IN A SECURITY VAULT FOR ANY FURTHER INVESTMENT PERHAPS IN YOUR COUNTRY.\n","YOU WERE INTRODUCED TO ME BY A RELIABLE FRIEND OF MINE WHO IS A TRAVELLER,AND ALSO A MEMBER OF CHAMBER OF COMMERCE AS A RELIABLE AND TRUSTWORTHY PERSON WHOM I CAN RELY ON AS FOREIGN PARTNER, EVEN THOUGH THE NATURE OF THE TRANSACTION WAS NOT REVEALED TO HIM FOR SECURITY REASONS.\n","THE (USD$25M) WAS PART OF A PROCEEDS FROM DIAMOND TRADE MEANT FOR THE LATE PRESIDENT LAURENT KABILA WHICH WAS DELIVERED THROUGH ZIMBABWE IN DIPLOMATIC BOXES. THE BOXES WERE KEPT UNDER MY CUSTODY BEFORE THE SAD EVENT THAT TOOK THE LIFE OF (MR. PRESIDENT).THE CONFUSION THAT ENSUED AFTER THE ASSASSINATION AND THE SPORADIC SHOOTING AMONG THE FACTIONS, I HAVE TO RUN AWAY FROM THE COUNTRY FOR MY DEAR LIFE AS I AM NOT A SOLDIER BUT A CIVIL SERVANT I CROSSED RIVER CONGO TO OTHER SIDE OF CONGO LIBREVILLE FROM THERE I MOVED TO THE THIRD COUNTRY GHANA WHERE I AM PRESENTLY TAKING REFUGE.\n","AS A MATTER OF FACT, WHAT I URGENTLY NEEDED FROM YOU IS YOUR ASSISTANCE IN MOVING THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTRY FOR INVESTMENT WITHOUT RAISING EYEBROW. FOR YOUR ASSISTANCE I WILL GIVE YOU 20% OF THE TOTAL SUM AS YOUR OWN SHARE WHEN THE MONEY GETS TO YOUR ACCOUNT, WHILE 75% WILL BE FOR ME, OF WHICH WITH YOUR KIND ADVICE I HOPE TO INVEST IN PROFITABLE VENTURE IN YOUR COUNTRY IN OTHER TO SETTLE DOWN FOR MEANINGFUL LIFE, AS I AM TIRED OF LIVING IN A WAR ENVIRONMENT.\n","THE REMAINING 5% WILL BE USED TO OFFSET ANY COST INCURRED IN THE CAUSE OF MOVING THE MONEY TO YOUR ACCOUNT. IF THE PROPOSAL IS ACCEPTABLE TO YOU PLEASE CONTACT ME IMMEDIATELY THROUGH THE ABOVE TELEPHONE AND E-MAIL, TO ENABLE ME ARRANGE FACE TO FACE MEETING WITH YOU IN GHANA FOR THE CLEARANCE OF THE FUNDS BEFORE TRANSFRING IT TO YOUR BANK ACCOUNT AS SEEING IS BELIEVING.\n","FINALLY, IT IS IMPORTANT ALSO THAT I LET YOU UNDERSTAND THAT THERE IS NO RISK INVOLVED WHATSOEVER AS THE MONEY HAD NO RECORD IN KINSHASA FOR IT WAS MEANT FOR THE PERSONAL USE OF (MR. PRESIDEND ) BEFORE THE NEFARIOUS INCIDENT OCCURRED, AND ALSO I HAVE ALL THE NECESSARY DOCUMENTS AS REGARDS TO THE FUNDS INCLUDING THE (CERTIFICATE OF DEPOSIT), AS I AM THE DEPOSITOR OF THE CONSIGNMENT.\n","LOOKING FORWARD TO YOUR URGENT RESPONSE.YOUR SINCERELY,\tMR. JAMES NGOLA. \n","Number of 2 newlines 0\n","Number of tabs 1\n","Number of 2 tabs 0\n","Number of tab and newline 0\n","Regex text lenght 3213\n"]}]},{"cell_type":"markdown","source":["##Exercise\n","How many '\\n' can be found in the first 2000 chars of the movie dataset? "],"metadata":{"id":"xB5zI2kFQhgy"}},{"cell_type":"code","source":["raw[:2000].count('\\n')"],"metadata":{"id":"v20s0IyVf-ZM","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668007239694,"user_tz":-60,"elapsed":242,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"c215438e-f263-4fbd-dc46-ee4285c7a2d1"},"execution_count":59,"outputs":[{"output_type":"execute_result","data":{"text/plain":["16"]},"metadata":{},"execution_count":59}]},{"cell_type":"markdown","metadata":{"id":"k_X3rqVnKQ_n"},"source":["# Case folding\n","\n"]},{"cell_type":"code","metadata":{"id":"Rq5yIptLKRKV","colab":{"base_uri":"https://localhost:8080/"},"outputId":"c0d4a4e6-204b-403a-8530-5761a6159f45","executionInfo":{"status":"ok","timestamp":1668007275199,"user_tz":-60,"elapsed":854,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["print(\"Initial text:\")\n","print(text[:110])\n","lower = text.lower()\n","print()\n","print(\"Final text:\")\n","print(lower[:110])"],"execution_count":60,"outputs":[{"output_type":"stream","name":"stdout","text":["Initial text:\n","DEAR FRIEND,I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) \n","\n","Final text:\n","dear friend,i am ( dr.) james ngola, the personal assistance to the late congolese (president laurent kabila) \n"]}]},{"cell_type":"markdown","metadata":{"id":"IGH8wMm9KQpP"},"source":["# Remove punctuation and other Symbols"]},{"cell_type":"code","metadata":{"id":"1QB7V4LdKQxp","colab":{"base_uri":"https://localhost:8080/","height":252},"outputId":"3e9ac383-e992-4afe-8f1e-9456b5b228af","executionInfo":{"status":"ok","timestamp":1668007445193,"user_tz":-60,"elapsed":242,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["#library that contains punctuation\n","import string\n","print(\"Library's punctuation: \", string.punctuation)\n","#string.punctuation + '€'\n","### Customizing \n","custom = list(string.punctuation)\n","custom.append('€')\n","print('Custom list:',custom,'\\n')\n","\n","#defining the function to remove punctuation\n","def remove_punctuation(text):\n"," punctuationfree=\"\".join([i for i in text if i not in string.punctuation])\n"," return punctuationfree\n","\n","def custom_re(text):\n"," punctuationfree=\"\".join([i for i in text if i not in custom])\n"," return punctuationfree\n","\n","sm_text= 'Everything is ok! €€'\n","print(\"Simple Example:\",sm_text)\n","print(\"Original string list:\",remove_punctuation(sm_text))\n","print(\"Custom Removal:\",custom_re(sm_text))\n","\n","print(\"\\n Spam Example\")\n","#storing the puntuation free text\n","data_spam['v2'] = data_spam['v2'].apply(lambda x: remove_punctuation(x))\n","display(data_spam.head(2))"],"execution_count":62,"outputs":[{"output_type":"stream","name":"stdout","text":["Library's punctuation: !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n","Custom list: ['!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~', '€'] \n","\n","Simple Example: Everything is ok! €€\n","Original string list: Everything is ok €€\n","Custom Removal: Everything is ok \n","\n"," Spam Example\n"]},{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","\n"," v2 \n","0 Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat \n","1 Ok lar Joking wif u oni "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2
0hamGo until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat
1hamOk lar Joking wif u oni
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"3PekL0u0gbCY"},"source":["# Remove numbers \n"]},{"cell_type":"code","metadata":{"id":"NrjNJS3DgbKC","colab":{"base_uri":"https://localhost:8080/","height":358},"executionInfo":{"status":"ok","timestamp":1668007567452,"user_tz":-60,"elapsed":341,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"6b9b8c4b-2b8c-4fc0-a0ad-b27448e9be42"},"source":["n_text = \"In July 1776, during the American Revolutionary War, the 13 Colonies, acting jointly through the 2nd Continental..\"\n","print(n_text)\n","no_num = re.sub(r'\\d+', '', n_text)\n","print(no_num + '\\n')\n","\n","data_spam['v2']= data_spam['v2'].apply(lambda x: re.sub(r'\\d+', '', x))\n","display(data_spam.head(5))"],"execution_count":63,"outputs":[{"output_type":"stream","name":"stdout","text":["In July 1776, during the American Revolutionary War, the 13 Colonies, acting jointly through the 2nd Continental..\n","In July , during the American Revolutionary War, the Colonies, acting jointly through the nd Continental..\n","\n"]},{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","2 spam \n","3 ham \n","4 ham \n","\n"," v2 \n","0 Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat \n","1 Ok lar Joking wif u oni \n","2 Free entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs \n","3 U dun say so early hor U c already then say \n","4 Nah I dont think he goes to usf he lives around here though "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2
0hamGo until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat
1hamOk lar Joking wif u oni
2spamFree entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs
3hamU dun say so early hor U c already then say
4hamNah I dont think he goes to usf he lives around here though
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"ZOsbxHBtgbfA"},"source":["# Remove repeated characters \n"]},{"cell_type":"code","metadata":{"id":"I4vGsn6Zgbmj","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668007818240,"user_tz":-60,"elapsed":298,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"f651840e-9bbc-4916-f804-a8c6afe6b3c9"},"source":["def character_repeatation(text):\n"," # Pattern matching for all case alphabets\n"," # \\1 It refers to the first capturing group.\n"," # {2,} It means we are matching for repetition that occurs more than two times (or equal).\n"," # r’\\1\\1' → It limits all the repetition to two characters.\n"," Pattern_alpha = re.compile(r\"([A-Za-z])\\1{2,}\", re.DOTALL)\n"," # Limiting all the repeatation to two characters.\n"," Formatted_text = Pattern_alpha.sub(r\"\\1\\1\", text) \n"," # Pattern matching for all the punctuations that can occur\n"," Pattern_Punct = re.compile(r'([.,/#!$%^&*?;:{}=_`~()+-])\\1{1,}')\n"," # Limiting punctuations in previously formatted string to only one.\n"," Combined_Formatted = Pattern_Punct.sub(r'\\1', Formatted_text)\n"," return Combined_Formatted\n","\n","repeated = \"Realllllllllyyyyy good!!!!!\"\n","print(\"Before:\",repeated)\n","print(\"After:\",character_repeatation(repeated))"],"execution_count":64,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: Realllllllllyyyyy good!!!!!\n","After: Reallyy good!\n"]}]},{"cell_type":"markdown","metadata":{"id":"SYO0KuF-Gaz2"},"source":["# Spelling Correction"]},{"cell_type":"code","metadata":{"id":"NzYuuGb9Ga73","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668007889851,"user_tz":-60,"elapsed":1410,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"cfcdb2dc-76a5-42b4-9c71-bcc7486f84ac"},"source":["# !pip install textblob\n","%%timeit\n","from textblob import TextBlob\n","wrong = \"raeyly good!!!!!\"\n","\n","def correct_me(text):\n"," textBlb = TextBlob(text) \n"," textCorrected = textBlb.correct() # Correcting the text\n"," return textCorrected\n","\n","print(\"Before:\",wrong)\n","print(\"After:\",correct_me(wrong))"],"execution_count":67,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","Before: raeyly good!!!!!\n","After: really good!!!!!\n","187 ms ± 60.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"]}]},{"cell_type":"code","source":["# Example from the slides:\n","\n","wrong_slides = \"repeaaaaaatedddd !!!\"\n","\n","print(\"Before:\",wrong_slides)\n","print(\"After:\",correct_me(wrong_slides))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pRaX26-8DFzX","executionInfo":{"status":"ok","timestamp":1668007920199,"user_tz":-60,"elapsed":1057,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"df2e7989-97b1-4e82-bc05-629985ffcd64"},"execution_count":68,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: repeaaaaaatedddd !!!\n","After: repeaaaaaatedddd !!!\n"]}]},{"cell_type":"markdown","source":["Welp it did not correct! We need to combine it with the regex to remove multiple repetitions!"],"metadata":{"id":"9UBDRjhHDUCz"}},{"cell_type":"code","source":["print(\"Before:\", wrong_slides)\n","print(\"After repetition removal:\", character_repeatation(wrong_slides))\n","print(\"After repetition removal + correction:\", correct_me(character_repeatation(wrong_slides)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pXkfJwu-DgPV","executionInfo":{"status":"ok","timestamp":1668007952093,"user_tz":-60,"elapsed":798,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"d90f2bea-819a-421b-86aa-4420867bf9b7"},"execution_count":69,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: repeaaaaaatedddd !!!\n","After repetition removal: repeaatedd !\n","After repetition removal + correction: repeated !\n"]}]},{"cell_type":"markdown","metadata":{"id":"N6243JrHgbt5"},"source":["# Remove Links"]},{"cell_type":"code","metadata":{"id":"IrpMEctBgb1m","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668008020236,"user_tz":-60,"elapsed":228,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"283e4e8e-6c0e-47fc-aa29-3a205a784505"},"source":["def remove_links(text):\n"," # Removing all the occurrences of links that starts with https\n"," remove_https = re.sub(r'http\\S+', ' ', text)\n"," # Remove all the occurrences of text that ends with .com\n"," remove_com = re.sub(r\"www\\.[A-Za-z]*\\.com\", \" \", remove_https)\n"," return remove_com\n","\n","link_text = \"For more information visit www.example.com or https://www.example.com.\"\n","print(\"Before:\",link_text)\n","print(\"After:\",remove_links(link_text))"],"execution_count":70,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: For more information visit www.example.com or https://www.example.com but\n","After: For more information visit or but\n"]}]},{"cell_type":"markdown","source":["Remember during the slides presentation we talked how there can a lot of issues with the presented regexes, well here is an example where it fails:"],"metadata":{"id":"A07Nn5NCnR6j"}},{"cell_type":"code","source":["link_text = \"Download mongoDB from mongodb.com and access it from 127.0.0.0:27020 192.168.1.1\"\n","print(\"Before:\",link_text)\n","print(\"After:\",remove_links(link_text))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"W2hSolkcnRA3","executionInfo":{"status":"ok","timestamp":1668008036841,"user_tz":-60,"elapsed":238,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"c33d4bef-6f89-4a0f-9630-b145e3b3c34b"},"execution_count":71,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: Download mongoDB from mongodb.com and access it from 127.0.0.0:27020\n","After: Download mongoDB from mongodb.com and access it from 127.0.0.0:27020\n"]}]},{"cell_type":"markdown","source":["## Exercise: \n","Think about a regex to solve the mentioned problems."],"metadata":{"id":"S4MNBzg_n7ww"}},{"cell_type":"code","source":["link_text = \"Download mongoDB from mongodb.com and access it from 127.0.0.0:27020 192.168.1.255\"\n","\n","def remove_links_improved(text):\n"," # do something here\n"," text = re.sub(r'[A-Za-z]*\\.com', '', text)\n"," text = re.sub(r'\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d+', '', text)\n"," text = re.sub(r'\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}', '', text)\n"," return text\n"," #raise NotImplementedError('Implement the function before')\n","\n","print(\"Before:\",link_text)\n","print(\"After:\",remove_links_improved(link_text))"],"metadata":{"id":"SkV69B_dn7gA","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668009038861,"user_tz":-60,"elapsed":8,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"9cb6afee-3097-4665-a47a-a4f677200f40"},"execution_count":76,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: Download mongoDB from mongodb.com and access it from 127.0.0.0:27020 192.168.1.255\n","After: Download mongoDB from and access it from \n"]}]},{"cell_type":"markdown","metadata":{"id":"xn7lrvFbOaOf"},"source":["# Removing Emoji"]},{"cell_type":"code","source":["!pip install emoji\n","!pip install demoji"],"metadata":{"id":"JUvRw_34LYvU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668009226933,"user_tz":-60,"elapsed":6985,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"8dd7c6ae-1c31-402b-dd37-9b5a2d6a4ccb"},"execution_count":77,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: emoji in /usr/local/lib/python3.7/dist-packages (2.2.0)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: demoji in /usr/local/lib/python3.7/dist-packages (1.1.0)\n"]}]},{"cell_type":"code","metadata":{"id":"6wOpGlV8Oab5","colab":{"base_uri":"https://localhost:8080/"},"outputId":"80f2482d-456b-4546-a94e-3b5b40ee9b1a","executionInfo":{"status":"ok","timestamp":1668009278807,"user_tz":-60,"elapsed":265,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["import emoji\n","import demoji\n","\n","print(\"\\U0001f605 is \\\\U0001f605 in utf\")\n","\n","print('\\U0001f605 is :grinning_face_with_sweat: in emoji library')\n","print(emoji.emojize('emojized :grinning_face_with_sweat:'))"],"execution_count":78,"outputs":[{"output_type":"stream","name":"stdout","text":["😅 is \\U0001f605 in utf\n","😅 is :grinning_face_with_sweat: in emoji library\n","emojized 😅\n"]}]},{"cell_type":"code","source":["print(\"\\n Working with Tweets\")\n","tweet = \"\"\"\\\n","... #startspreadingthenews yankees win great start by 🎅🏾 going 5strong innings with 5k’s🔥 🐂\n","... solo homerun 🌋 🇲🇽 and 🇳🇮 to close the game🔥🔥!!!....\n","... WHAT A GAME!!\"\"\"\n","\n","print(tweet)\n","display(demoji.findall(tweet))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":214},"id":"fvvBmFTD1W11","executionInfo":{"status":"ok","timestamp":1667999968416,"user_tz":-60,"elapsed":573,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"101c5ebc-da47-400c-932a-21f603fc3dfe"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n"," Working with Tweets\n","... #startspreadingthenews yankees win great start by 🎅🏾 going 5strong innings with 5k’s🔥 🐂\n","... solo homerun 🌋 🇲🇽 and 🇳🇮 to close the game🔥🔥!!!....\n","... WHAT A GAME!!\n"]},{"output_type":"display_data","data":{"text/plain":["{'🔥': 'fire',\n"," '🐂': 'ox',\n"," '🌋': 'volcano',\n"," '🎅🏾': 'Santa Claus: medium-dark skin tone',\n"," '🇳🇮': 'flag: Nicaragua',\n"," '🇲🇽': 'flag: Mexico'}"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"apoWA5a6grJU"},"source":["# Remove HTML Tags\n"]},{"cell_type":"code","metadata":{"id":"MTN9qI5rgrQm","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668009480688,"user_tz":-60,"elapsed":229,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"83961c06-90e6-4393-f88d-649e8aa5c953"},"source":["from bs4 import BeautifulSoup \n","\n","sample = 'This is a nice place to live. Image description'\n","\n","def strip_html_tags(text):\n"," # Initiating BeautifulSoup object soup.\n"," soup = BeautifulSoup(text, \"html.parser\")\n"," # Get all the text other than html tags.\n"," stripped_text = soup.get_text(separator=\" \")\n"," return stripped_text\n","\n","print(\"Before:\",sample)\n","print('After:', strip_html_tags(sample))"],"execution_count":79,"outputs":[{"output_type":"stream","name":"stdout","text":["Before: This is a nice place to live. Image description\n","After: This is a nice place to live. Image description\n"]}]},{"cell_type":"markdown","metadata":{"id":"hTv8Qld5jYUM"},"source":["# TIME for Exercise 1.1\n","\n"]},{"cell_type":"markdown","metadata":{"id":"R3fjGXO7usOS"},"source":["## 1.1\n","\n","Use the data stored in the varaible **raw**. \n","\n","1. Remove whitespaces,numbers, punctuation and lower the raw_text. Save as new_text.\n","\n","There is a text sample in the hidden cell named evaluation_text:\n","\n",">A. How many times does the number 1 appears in the evaluation_text? \\\\\n",">B. Can you find the € symbol in the evaluation_text? \\\\\n",">C. If B == True, then remove it. \\\\\n",">D. How many times does the number 1 appears in the raw_text? \\\\\n",">E. Remove numbers from raw_text.\n","```\n","You can use:\n","import string\n","import regex as re\n","text.replace('', '')\n","text.count(' ')\n","\"\".join([character for character in text if character not in string.punctuation])+\n","```"]},{"cell_type":"code","source":[],"metadata":{"id":"jOw3rXsgJsgX"},"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-JYmzyeajo3r","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668010064208,"user_tz":-60,"elapsed":2436,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"6974c877-555f-48c1-da41-c1bcada7efb6"},"source":["evaluation_text = \"\"\"it's everything a movie can be : funny , dramatic , interesting , weird , funny , weird and strikingly original . \n","yep that pretty much describes this movie . \n","it starts out like a regular movie and ends up being one of the weirdest , funniest most original movies i have ever seen . \n","it boggles the mind and some have to wonder why we cannot get €€ movies like this more often . \n","besides being one of the best films of the year , being john malkovich may as well be one of the best movies ever . \n","period . \n","then again there are so many good movies , that one cannot pick an all time favorite . \n","john cusack plays a puppeteer craig schwartz a man out of 1 a job , in search of a job . \n","his wife lotte schwartz who is played by a completely un-noticeable cameron diaz who looks like something off the streets is an animal lover and has about every kind of animal you could think of . \n","craig finds a job as a file at the 7 ? floor of a business€ building . . . you \n","have to pry 1 open the elevator doors open before it reaches floor eight , the 7 ? floor is just a floor between 7 and 8 . \n","he is hired by his 105 year old boss ( orsen bean ) to be a filer . \n","in his office , he discovers a little door €, to which was boarded up and hidden . \n","to his curiosity he opens it and starts to 11 crawl toward it , he then gets sucked to the end and ends up in john malkovich's mind . \n","fifteen minutes later he is shot out onto the side of the new jersey turnpike . \n","he returns to tell his co-worker maxine ( catherine keener ) that he has found a portal that will lead him into john malkovich's mind , she doesn't believe him but after she sees it it changes her mind . \n","lotte also finds out about the portal and discovers that being someone else is good after all . \n","john malkovich of course has no idea what is going on , and by the end of this bizarre film there are so many twists and turns , that we don't know what really happened . \n","john cusack is outstanding and utterly believeable in a role only he himself could play . \n","he fits the role perfectly and to me was brilliantly cast . \n","cameron diaz is outstanding and utterly one of the world's most prettiest women , is made up here unnoticeable and very unattractive . \n","she however gives a very comic performance and this could easily be her best role to date . \n","catherine keener is very funny and sexy as maxine and of course the best thing of the movie is the magic himself john malkovich who is very brilliant and this movie plays big time homage ? to the master himself . \n","the whole group combined gives us a wonderfully funny movie that is also smart and clever . \"\"\"\n","\n","# add your code here\n","import string\n","import regex as re\n","def remove_punctuation(text):\n"," punctuationfree=\"\".join([i for i in text if i not in string.punctuation])\n"," return punctuationfree\n"," \n","new_text_ex = re.sub(r'\\s\\s', ' ', raw)\n","new_text_ex = re.sub(r'\\d+', ' ', raw)\n","new_text_ex = remove_punctuation(new_text_ex)\n","new_text_ex = new_text_ex.lower()\n"," \n","print(evaluation_text.count('1')) #5\n","print(evaluation_text.count('€')) #4\n","print(evaluation_text.replace('€', ''))\n","print(raw.count('1')) #3288\n","\n","x = re.sub(r'\\d', '', raw)\n","print(x.count('1')) #0\n"],"execution_count":80,"outputs":[{"output_type":"stream","name":"stdout","text":["5\n","4\n","3288\n","0\n"]}]},{"cell_type":"markdown","metadata":{"id":"a1Cw5aI6gt3V"},"source":["# Tokenization"]},{"cell_type":"code","metadata":{"id":"4uqsALwVgt-n","colab":{"base_uri":"https://localhost:8080/"},"outputId":"559336e7-c41a-417a-a831-2c169425e7c6","executionInfo":{"status":"ok","timestamp":1668011994424,"user_tz":-60,"elapsed":228,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["#defining function for tokenization\n","tokenize = text[:111]\n","print('Before',re.sub(r'\\s\\s', '', tokenize))\n","\n","print('\\nString Operator', tokenize.split())\n","\n","import re\n","def tokenization(text):\n"," tokens = re.split('\\W+',text)\n"," return tokens\n","print(\"\\nSimple Regex:\",tokenization(tokenize))\n","\n","\n","###WordPunctTokenizer\n","from nltk.tokenize import WordPunctTokenizer\n","new_text4 = WordPunctTokenizer().tokenize(tokenize)\n","print(\"\\nWordPunctTokenizer:\",new_text4)\n","\n","print('\\nWorking with Tweets\\n')\n","###TweetTokenizer\n","from nltk.tokenize import TweetTokenizer\n","tknzr = TweetTokenizer()\n","tweet_tok = \"This is a cooool #dummysmiley: :-) :-P <3 and some arrows < > -> <--\"\n","new_tweet = tknzr.tokenize(tweet_tok)\n","print('TweetTokenizer:', new_tweet)\n","print(\"\\nSimple Regex:\",tokenization(tweet_tok))"],"execution_count":81,"outputs":[{"output_type":"stream","name":"stdout","text":["Before DEAR FRIEND,I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) W\n","\n","String Operator ['DEAR', 'FRIEND,I', 'AM', '(', 'DR.)', 'JAMES', 'NGOLA,', 'THE', 'PERSONAL', 'ASSISTANCE', 'TO', 'THE', 'LATE', 'CONGOLESE', '(PRESIDENT', 'LAURENT', 'KABILA)', 'W']\n","\n","Simple Regex: ['DEAR', 'FRIEND', 'I', 'AM', 'DR', 'JAMES', 'NGOLA', 'THE', 'PERSONAL', 'ASSISTANCE', 'TO', 'THE', 'LATE', 'CONGOLESE', 'PRESIDENT', 'LAURENT', 'KABILA', 'W']\n","\n","WordPunctTokenizer: ['DEAR', 'FRIEND', ',', 'I', 'AM', '(', 'DR', '.)', 'JAMES', 'NGOLA', ',', 'THE', 'PERSONAL', 'ASSISTANCE', 'TO', 'THE', 'LATE', 'CONGOLESE', '(', 'PRESIDENT', 'LAURENT', 'KABILA', ')', 'W']\n","\n","Working with Tweets\n","\n","TweetTokenizer: ['This', 'is', 'a', 'cooool', '#dummysmiley', ':', ':-)', ':-P', '<3', 'and', 'some', 'arrows', '<', '>', '->', '<--']\n","\n","Simple Regex: ['This', 'is', 'a', 'cooool', 'dummysmiley', 'P', '3', 'and', 'some', 'arrows', '']\n"]}]},{"cell_type":"markdown","metadata":{"id":"uEraz5_Qgw9q"},"source":["# Sentence split\n"]},{"cell_type":"code","metadata":{"id":"ml4siZ3AgxDx","colab":{"base_uri":"https://localhost:8080/","height":399},"outputId":"9a443f3d-7caa-4277-e6cd-1d3c2bebf298","executionInfo":{"status":"ok","timestamp":1666529281949,"user_tz":-120,"elapsed":256,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["from nltk.tokenize import sent_tokenize, word_tokenize\n","new_text3 = sent_tokenize(text)\n","display(new_text3)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["['DEAR FRIEND,I AM ( DR.) JAMES NGOLA, THE PERSONAL ASSISTANCE TO THE LATE CONGOLESE (PRESIDENT LAURENT KABILA) WHO WAS ASSASSINATED BY HIS BODY GUARD ON 16TH JAN. 2001.',\n"," 'THE INCIDENT OCCURRED IN OUR PRESENCE WHILE WE WERE HOLDING MEETING WITH HIS EXCELLENCY OVER THE FINANCIAL RETURNS FROM THE DIAMOND SALES IN THE AREAS CONTROLLED BY (D.R.C.)',\n"," 'DEMOCRATIC REPUBLIC OF CONGO FORCES AND THEIR FOREIGN ALLIES ANGOLA AND ZIMBABWE, HAVING RECEIVED THE PREVIOUS DAY (USD$100M) ONE HUNDRED MILLION UNITED STATES DOLLARS, CASH IN THREE DIPLOMATIC BOXES ROUTED THROUGH ZIMBABWE.MY PURPOSE OF WRITING YOU THIS LETTER IS TO SOLICIT FOR YOUR ASSISTANCE AS TO BE A COVER TO THE FUND AND ALSO COLLABORATION IN MOVING THE SAID FUND INTO YOUR BANK ACCOUNT THE SUM OF (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY, WHICH I DEPOSITED WITH A SECURITY COMPANY IN GHANA, IN A DIPLOMATIC BOX AS GOLDS WORTH (USD$25M) TWENTY FIVE MILLION UNITED STATES DOLLARS ONLY FOR SAFE KEEPING IN A SECURITY VAULT FOR ANY FURTHER INVESTMENT PERHAPS IN YOUR COUNTRY.',\n"," 'YOU WERE INTRODUCED TO ME BY A RELIABLE FRIEND OF MINE WHO IS A TRAVELLER,AND ALSO A MEMBER OF CHAMBER OF COMMERCE AS A RELIABLE AND TRUSTWORTHY PERSON WHOM I CAN RELY ON AS FOREIGN PARTNER, EVEN THOUGH THE NATURE OF THE TRANSACTION WAS NOT REVEALED TO HIM FOR SECURITY REASONS.',\n"," 'THE (USD$25M) WAS PART OF A PROCEEDS FROM DIAMOND TRADE MEANT FOR THE LATE PRESIDENT LAURENT KABILA WHICH WAS DELIVERED THROUGH ZIMBABWE IN DIPLOMATIC BOXES.',\n"," 'THE BOXES WERE KEPT UNDER MY CUSTODY BEFORE THE SAD EVENT THAT TOOK THE LIFE OF (MR. PRESIDENT).THE CONFUSION THAT ENSUED AFTER THE ASSASSINATION AND THE SPORADIC SHOOTING AMONG THE FACTIONS, I HAVE TO RUN AWAY FROM THE COUNTRY FOR MY DEAR LIFE AS I AM NOT A SOLDIER BUT A CIVIL SERVANT I CROSSED RIVER CONGO TO OTHER SIDE OF CONGO LIBREVILLE FROM THERE I MOVED TO THE THIRD COUNTRY GHANA WHERE I AM PRESENTLY TAKING REFUGE.',\n"," 'AS A MATTER OF FACT, WHAT I URGENTLY NEEDED FROM YOU IS YOUR ASSISTANCE IN MOVING THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTRY FOR INVESTMENT WITHOUT RAISING EYEBROW.',\n"," 'FOR YOUR ASSISTANCE I WILL GIVE YOU 20% OF THE TOTAL SUM AS YOUR OWN SHARE WHEN THE MONEY GETS TO YOUR ACCOUNT, WHILE 75% WILL BE FOR ME, OF WHICH WITH YOUR KIND ADVICE I HOPE TO INVEST IN PROFITABLE VENTURE IN YOUR COUNTRY IN OTHER TO SETTLE DOWN FOR MEANINGFUL LIFE, AS I AM TIRED OF LIVING IN A WAR ENVIRONMENT.',\n"," 'THE REMAINING 5% WILL BE USED TO OFFSET ANY COST INCURRED IN THE CAUSE OF MOVING THE MONEY TO YOUR ACCOUNT.',\n"," 'IF THE PROPOSAL IS ACCEPTABLE TO YOU PLEASE CONTACT ME IMMEDIATELY THROUGH THE ABOVE TELEPHONE AND E-MAIL, TO ENABLE ME ARRANGE FACE TO FACE MEETING WITH YOU IN GHANA FOR THE CLEARANCE OF THE FUNDS BEFORE TRANSFRING IT TO YOUR BANK ACCOUNT AS SEEING IS BELIEVING.',\n"," 'FINALLY, IT IS IMPORTANT ALSO THAT I LET YOU UNDERSTAND THAT THERE IS NO RISK INVOLVED WHATSOEVER AS THE MONEY HAD NO RECORD IN KINSHASA FOR IT WAS MEANT FOR THE PERSONAL USE OF (MR. PRESIDEND ) BEFORE THE NEFARIOUS INCIDENT OCCURRED, AND ALSO I HAVE ALL THE NECESSARY DOCUMENTS AS REGARDS TO THE FUNDS INCLUDING THE (CERTIFICATE OF DEPOSIT), AS I AM THE DEPOSITOR OF THE CONSIGNMENT.',\n"," 'LOOKING FORWARD TO YOUR URGENT RESPONSE.YOUR SINCERELY,\\tMR. JAMES NGOLA.']"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"wEUciAKGgzld"},"source":["# Remove stop words"]},{"cell_type":"code","source":["import nltk\n","nltk.download('stopwords')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"wi4y_fCU4164","executionInfo":{"status":"ok","timestamp":1668012085751,"user_tz":-60,"elapsed":6,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"cb5b441a-30ac-43b6-a4ac-677efb63f924"},"execution_count":82,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data] Unzipping corpora/stopwords.zip.\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":82}]},{"cell_type":"code","metadata":{"id":"TtmEgZWjgztU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668012250721,"user_tz":-60,"elapsed":4772,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"11012219-6597-46f9-8e81-bbfda68fc044"},"source":["stop_words = nltk.corpus.stopwords.words('english')\n","\n","def remove_stopwords(text):\n"," tokenized_text = WordPunctTokenizer().tokenize(text)\n","\n"," tokenized_text_without_stopwords = []\n"," for token in tokenized_text:\n"," if token.lower() not in stop_words:\n"," tokenized_text_without_stopwords.append(token)\n"," return tokenized_text_without_stopwords\n","\n","stop_me = \"We are ready to travel to NY.\"\n","print('With Stopwords:', (stop_me.split()))\n","print('Without Stopwords:', remove_stopwords(stop_me))\n","\n","#Remove stopwords from the movie reviews\n","print('Movie Reviews: Total tokens with stopwords:',len(WordPunctTokenizer().tokenize(raw)))\n","clean_re = remove_stopwords(raw)\n","print('Movie Reviews: Total tokens without stopwords:',len(clean_re))\n","\n"],"execution_count":84,"outputs":[{"output_type":"stream","name":"stdout","text":["With Stopwords: ['We', 'are', 'ready', 'to', 'travel', 'to', 'NY.']\n","Without Stopwords: ['ready', 'travel', 'NY', '.']\n","Movie Reviews: Total tokens with stopwords: 1583820\n","Movie Reviews: Total tokens without stopwords: 955610\n"]}]},{"cell_type":"markdown","metadata":{"id":"Vw1dRElGg2er"},"source":["# Stemming"]},{"cell_type":"code","metadata":{"id":"9-1CfwlHg2oE","colab":{"base_uri":"https://localhost:8080/","height":285},"outputId":"d027071e-499d-4932-b7c8-6ea17d666d1e","executionInfo":{"status":"ok","timestamp":1668012342336,"user_tz":-60,"elapsed":2857,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["#importing the Stemming function from nltk library\n","from nltk.stem.porter import PorterStemmer\n","#defining the object for stemming\n","porter_stemmer = PorterStemmer()\n","\n","#defining a function for stemming\n","def stemming(text):\n"," stem_text = [porter_stemmer.stem(word) for word in text]\n"," return stem_text\n"," \n","data_spam['no_stopwords']= data_spam['v2'].apply(lambda x: remove_stopwords(x))\n","\n","data_spam['msg_stemmed']=data_spam['no_stopwords'].apply(lambda x: stemming(x))\n","display(data_spam.head(5))"],"execution_count":86,"outputs":[{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","2 spam \n","3 ham \n","4 ham \n","\n"," v2 \\\n","0 Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat \n","1 Ok lar Joking wif u oni \n","2 Free entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs \n","3 U dun say so early hor U c already then say \n","4 Nah I dont think he goes to usf he lives around here though \n","\n"," no_stopwords \\\n","0 [Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat] \n","1 [Ok, lar, Joking, wif, u, oni] \n","2 [Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, overs] \n","3 [U, dun, say, early, hor, U, c, already, say] \n","4 [Nah, dont, think, goes, usf, lives, around, though] \n","\n"," msg_stemmed \n","0 [go, jurong, point, crazi, avail, bugi, n, great, world, la, e, buffet, cine, got, amor, wat] \n","1 [ok, lar, joke, wif, u, oni] \n","2 [free, entri, wkli, comp, win, fa, cup, final, tkt, st, may, text, fa, receiv, entri, questionstd, txt, ratetc, appli, over] \n","3 [u, dun, say, earli, hor, u, c, alreadi, say] \n","4 [nah, dont, think, goe, usf, live, around, though] "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2no_stopwordsmsg_stemmed
0hamGo until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat[Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat][go, jurong, point, crazi, avail, bugi, n, great, world, la, e, buffet, cine, got, amor, wat]
1hamOk lar Joking wif u oni[Ok, lar, Joking, wif, u, oni][ok, lar, joke, wif, u, oni]
2spamFree entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs[Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, overs][free, entri, wkli, comp, win, fa, cup, final, tkt, st, may, text, fa, receiv, entri, questionstd, txt, ratetc, appli, over]
3hamU dun say so early hor U c already then say[U, dun, say, early, hor, U, c, already, say][u, dun, say, earli, hor, u, c, alreadi, say]
4hamNah I dont think he goes to usf he lives around here though[Nah, dont, think, goes, usf, lives, around, though][nah, dont, think, goe, usf, live, around, though]
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"cqfUPnqdg365"},"source":["# Lemmatization "]},{"cell_type":"code","source":["nltk.download('wordnet')\n","nltk.download('omw-1.4')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kVGeAUem5GHX","executionInfo":{"status":"ok","timestamp":1668012377549,"user_tz":-60,"elapsed":260,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"f04ae076-7b27-43d2-98b8-96aeeab637a5"},"execution_count":87,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":87}]},{"cell_type":"code","metadata":{"id":"jRlBzK2Pg4CF","colab":{"base_uri":"https://localhost:8080/","height":320},"executionInfo":{"status":"ok","timestamp":1668012433810,"user_tz":-60,"elapsed":2413,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"38102e68-9225-474b-9ecf-95c22dedaa7d"},"source":["from nltk.stem import WordNetLemmatizer\n","#defining the object for Lemmatization\n","wordnet_lemmatizer = WordNetLemmatizer()\n","\n","#defining the function for lemmatization\n","def lemmatizer(text):\n"," lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text]\n"," return lemm_text\n","\n","data_spam['msg_lemmatized']= data_spam['no_stopwords'].apply(lambda x: lemmatizer(x))\n","display(data_spam.head())"],"execution_count":88,"outputs":[{"output_type":"display_data","data":{"text/plain":[" v1 \\\n","0 ham \n","1 ham \n","2 spam \n","3 ham \n","4 ham \n","\n"," v2 \\\n","0 Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat \n","1 Ok lar Joking wif u oni \n","2 Free entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs \n","3 U dun say so early hor U c already then say \n","4 Nah I dont think he goes to usf he lives around here though \n","\n"," no_stopwords \\\n","0 [Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat] \n","1 [Ok, lar, Joking, wif, u, oni] \n","2 [Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, overs] \n","3 [U, dun, say, early, hor, U, c, already, say] \n","4 [Nah, dont, think, goes, usf, lives, around, though] \n","\n"," msg_stemmed \\\n","0 [go, jurong, point, crazi, avail, bugi, n, great, world, la, e, buffet, cine, got, amor, wat] \n","1 [ok, lar, joke, wif, u, oni] \n","2 [free, entri, wkli, comp, win, fa, cup, final, tkt, st, may, text, fa, receiv, entri, questionstd, txt, ratetc, appli, over] \n","3 [u, dun, say, earli, hor, u, c, alreadi, say] \n","4 [nah, dont, think, goe, usf, live, around, though] \n","\n"," msg_lemmatized \n","0 [Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat] \n","1 [Ok, lar, Joking, wif, u, oni] \n","2 [Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, over] \n","3 [U, dun, say, early, hor, U, c, already, say] \n","4 [Nah, dont, think, go, usf, life, around, though] "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
v1v2no_stopwordsmsg_stemmedmsg_lemmatized
0hamGo until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat[Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat][go, jurong, point, crazi, avail, bugi, n, great, world, la, e, buffet, cine, got, amor, wat][Go, jurong, point, crazy, Available, bugis, n, great, world, la, e, buffet, Cine, got, amore, wat]
1hamOk lar Joking wif u oni[Ok, lar, Joking, wif, u, oni][ok, lar, joke, wif, u, oni][Ok, lar, Joking, wif, u, oni]
2spamFree entry in a wkly comp to win FA Cup final tkts st May Text FA to to receive entry questionstd txt rateTCs apply overs[Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, overs][free, entri, wkli, comp, win, fa, cup, final, tkt, st, may, text, fa, receiv, entri, questionstd, txt, ratetc, appli, over][Free, entry, wkly, comp, win, FA, Cup, final, tkts, st, May, Text, FA, receive, entry, questionstd, txt, rateTCs, apply, over]
3hamU dun say so early hor U c already then say[U, dun, say, early, hor, U, c, already, say][u, dun, say, earli, hor, u, c, alreadi, say][U, dun, say, early, hor, U, c, already, say]
4hamNah I dont think he goes to usf he lives around here though[Nah, dont, think, goes, usf, lives, around, though][nah, dont, think, goe, usf, live, around, though][Nah, dont, think, go, usf, life, around, though]
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"GtlV4phEj_42"},"source":["# TIME for another Exercise "]},{"cell_type":"markdown","metadata":{"id":"Kd8IHATPWbQF"},"source":["## 1.2 \n","Use any tokenization method, you prefer to tokenize the **raw** and the **new_text**. \n",">A. How many tokens do they have? \\\\\n",">B. How many sentences do they have? \\\\ \n",">C. Apply **Stemming** and **Lemmatization** and then count the **tokens**. \n","\n","```\n","# You can use:\n","from nltk.tokenize import WordPunctTokenizer\n","WordPunctTokenizer().tokenize(tokenize)\n","from nltk.tokenize import TweetTokenizer\n","from nltk.stem import WordNetLemmatizer\n","nltk.download('wordnet')\n","from nltk.stem.porter import PorterStemmer\n","#defining the object for stemming\n","porter_stemmer = PorterStemmer()\n","\n","#For unique members of a list use set(list_name)\n","```\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"8unMLYynkYvY","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668012676535,"user_tz":-60,"elapsed":60238,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"9c31a28d-c56b-49a9-900d-f3618b1faad2"},"source":["# add your code here\n","\n","from nltk.tokenize import WordPunctTokenizer\n"," \n","tok_raw = WordPunctTokenizer().tokenize(raw)\n","print(\"\\nWordPunctTokenizer #Raw Total Tokens:\", len((tok_raw)))\n","print(\"\\nWordPunctTokenizer #Raw Unique Tokens:\", len(set(tok_raw)))\n"," \n","tok_new = WordPunctTokenizer().tokenize(new_text_ex)\n","print(\"\\nWordPunctTokenizer #New Total Tokens:\", len((tok_new)))\n","print(\"\\nWordPunctTokenizer #New Unique Tokens:\", len(set(tok_new)))\n","\n","print('\\n First 100 tokens for the raw and the preprocessed text:')\n","print(tok_raw[:100])\n","print(tok_new[:100])\n"," \n","from nltk.stem.porter import PorterStemmer\n","porter_stemmer = PorterStemmer()\n","def stemming(text):\n"," stem_text = [porter_stemmer.stem(word) for word in text]\n"," return stem_text\n","\n","from nltk.stem import WordNetLemmatizer\n","# nltk.download('wordnet')\n","# wordnet_lemmatizer = WordNetLemmatizer()\n"," \n","def lemmatizer(text):\n"," lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text]\n"," return lemm_text\n","\n","print(\"\\Stemmed #Raw Unique #Tokens:\", len(set(stemming((tok_raw)))))\n","print(\"\\Lemma #Raw Unique #Tokens:\", len(set(lemmatizer((tok_raw)))))\n","\n","print(\"\\Lemma #New Unique #Tokens:\", len(set(lemmatizer((tok_new)))))\n","print(\"\\Stemmed #New Unique #Tokens:\", len(set(stemming((tok_new)))))\n","\n","\n","\n","\n","\n","\n","\n","\n","\n","\n"],"execution_count":89,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","WordPunctTokenizer #Raw Total Tokens: 1583820\n","\n","WordPunctTokenizer #Raw Unique Tokens: 39768\n","\n","WordPunctTokenizer #New Total Tokens: 1293366\n","\n","WordPunctTokenizer #New Unique Tokens: 47531\n","\n"," First 100 tokens for the raw and the preprocessed text:\n","['plot', ':', 'two', 'teen', 'couples', 'go', 'to', 'a', 'church', 'party', ',', 'drink', 'and', 'then', 'drive', '.', 'they', 'get', 'into', 'an', 'accident', '.', 'one', 'of', 'the', 'guys', 'dies', ',', 'but', 'his', 'girlfriend', 'continues', 'to', 'see', 'him', 'in', 'her', 'life', ',', 'and', 'has', 'nightmares', '.', 'what', \"'\", 's', 'the', 'deal', '?', 'watch', 'the', 'movie', 'and', '\"', 'sorta', '\"', 'find', 'out', '.', '.', '.', 'critique', ':', 'a', 'mind', '-', 'fuck', 'movie', 'for', 'the', 'teen', 'generation', 'that', 'touches', 'on', 'a', 'very', 'cool', 'idea', ',', 'but', 'presents', 'it', 'in', 'a', 'very', 'bad', 'package', '.', 'which', 'is', 'what', 'makes', 'this', 'review', 'an', 'even', 'harder', 'one', 'to']\n","['plot', 'two', 'teen', 'couples', 'go', 'to', 'a', 'church', 'party', 'drink', 'and', 'then', 'drive', 'they', 'get', 'into', 'an', 'accident', 'one', 'of', 'the', 'guys', 'dies', 'but', 'his', 'girlfriend', 'continues', 'to', 'see', 'him', 'in', 'her', 'life', 'and', 'has', 'nightmares', 'whats', 'the', 'deal', 'watch', 'the', 'movie', 'and', 'sorta', 'find', 'out', 'critique', 'a', 'mindfuck', 'movie', 'for', 'the', 'teen', 'generation', 'that', 'touches', 'on', 'a', 'very', 'cool', 'idea', 'but', 'presents', 'it', 'in', 'a', 'very', 'bad', 'package', 'which', 'is', 'what', 'makes', 'this', 'review', 'an', 'even', 'harder', 'one', 'to', 'write', 'since', 'i', 'generally', 'applaud', 'films', 'which', 'attempt', 'to', 'break', 'the', 'mold', 'mess', 'with', 'your', 'head', 'and', 'such', 'lost', 'highway']\n","\\Stemmed #Raw Unique #Tokens: 26235\n","\\Lemma #Raw Unique #Tokens: 35334\n","\\Lemma #New Unique #Tokens: 42358\n","\\Stemmed #New Unique #Tokens: 32017\n"]}]},{"cell_type":"markdown","metadata":{"id":"MFRlKPAPg8B6"},"source":["# Part-of-speech Tagging"]},{"cell_type":"code","source":["nltk.download('averaged_perceptron_tagger')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SHHLMa6i5_6I","executionInfo":{"status":"ok","timestamp":1668013086627,"user_tz":-60,"elapsed":314,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"9219636c-aa3b-4a19-d83b-6a10b134db04"},"execution_count":90,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package averaged_perceptron_tagger to\n","[nltk_data] /root/nltk_data...\n","[nltk_data] Unzipping taggers/averaged_perceptron_tagger.zip.\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":90}]},{"cell_type":"code","metadata":{"id":"6MFqDjZ4g8Le","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668013121035,"user_tz":-60,"elapsed":808,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"cc958abd-104f-4999-bef6-73720e737696"},"source":["from textblob import TextBlob \n","\n","tag = [\"Sukanya, Rajib and Naba are my good friends. \" ,\n"," \"Sukanya is getting married next year. \" ,\n"," \"Marriage is a big step in one’s life.\" ,\n"," \"It is both exciting and frightening. \" ,\n"," \"But friendship is a sacred bond between people.\" ,\n"," \"It is a special kind of love between us. \",\n"," \"Many of you must have tried searching for a friend \",\n"," \"but never found the right one.\"]\n","for el in tag: \n"," result = TextBlob(el)\n"," print('\\n',el)\n"," print(result.tags)\n","\n","# NNP proper noun, singular ‘Harrison’\n","# CC coordinating conjunction\n","# VB verbs\n","# JJ adjective ‘big’"],"execution_count":91,"outputs":[{"output_type":"stream","name":"stdout","text":["\n"," Sukanya, Rajib and Naba are my good friends. \n","[('Sukanya', 'NNP'), ('Rajib', 'NNP'), ('and', 'CC'), ('Naba', 'NNP'), ('are', 'VBP'), ('my', 'PRP$'), ('good', 'JJ'), ('friends', 'NNS')]\n","\n"," Sukanya is getting married next year. \n","[('Sukanya', 'NNP'), ('is', 'VBZ'), ('getting', 'VBG'), ('married', 'VBN'), ('next', 'JJ'), ('year', 'NN')]\n","\n"," Marriage is a big step in one’s life.\n","[('Marriage', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('big', 'JJ'), ('step', 'NN'), ('in', 'IN'), ('one', 'CD'), ('’', 'NN'), ('s', 'JJ'), ('life', 'NN')]\n","\n"," It is both exciting and frightening. \n","[('It', 'PRP'), ('is', 'VBZ'), ('both', 'DT'), ('exciting', 'VBG'), ('and', 'CC'), ('frightening', 'NN')]\n","\n"," But friendship is a sacred bond between people.\n","[('But', 'CC'), ('friendship', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('sacred', 'JJ'), ('bond', 'NN'), ('between', 'IN'), ('people', 'NNS')]\n","\n"," It is a special kind of love between us. \n","[('It', 'PRP'), ('is', 'VBZ'), ('a', 'DT'), ('special', 'JJ'), ('kind', 'NN'), ('of', 'IN'), ('love', 'NN'), ('between', 'IN'), ('us', 'PRP')]\n","\n"," Many of you must have tried searching for a friend \n","[('Many', 'JJ'), ('of', 'IN'), ('you', 'PRP'), ('must', 'MD'), ('have', 'VB'), ('tried', 'VBN'), ('searching', 'VBG'), ('for', 'IN'), ('a', 'DT'), ('friend', 'NN')]\n","\n"," but never found the right one.\n","[('but', 'CC'), ('never', 'RB'), ('found', 'VBD'), ('the', 'DT'), ('right', 'JJ'), ('one', 'NN')]\n"]}]},{"cell_type":"markdown","metadata":{"id":"WFlg8mw1g8US"},"source":["# Term Dependencies\n"]},{"cell_type":"code","metadata":{"id":"pdClT5qqg8cD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1668013196721,"user_tz":-60,"elapsed":237,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"ef9e6f56-b472-4edc-bb8d-57d979776faa"},"source":["input_str = \"A black television and a white stove were bought for the new apartment of John.\"\n","result = TextBlob(input_str)\n","print(result.tags)\n","\n","# DT determiner\n","# JJ adjective ‘big’\n","#NN a noun"],"execution_count":92,"outputs":[{"output_type":"stream","name":"stdout","text":["[('A', 'DT'), ('black', 'JJ'), ('television', 'NN'), ('and', 'CC'), ('a', 'DT'), ('white', 'JJ'), ('stove', 'NN'), ('were', 'VBD'), ('bought', 'VBN'), ('for', 'IN'), ('the', 'DT'), ('new', 'JJ'), ('apartment', 'NN'), ('of', 'IN'), ('John', 'NNP')]\n"]}]},{"cell_type":"code","source":["reg_exp = \"NP: {
**}\"\n","rp = nltk.RegexpParser(reg_exp)\n","result = rp.parse(result.tags)\n","for el in result: \n"," print(el)\n","# print(result)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Qvc9_9Ui6Jou","executionInfo":{"status":"ok","timestamp":1668013255950,"user_tz":-60,"elapsed":276,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"80d978a0-10ec-4740-82c2-1f717b34a055"},"execution_count":93,"outputs":[{"output_type":"stream","name":"stdout","text":["(NP A/DT black/JJ television/NN)\n","('and', 'CC')\n","(NP a/DT white/JJ stove/NN)\n","('were', 'VBD')\n","('bought', 'VBN')\n","('for', 'IN')\n","(NP the/DT new/JJ apartment/NN)\n","('of', 'IN')\n","('John', 'NNP')\n"]}]},{"cell_type":"markdown","metadata":{"id":"hhEieKoIg8lg"},"source":["# Entity Recognition"]},{"cell_type":"code","source":["nltk.download('maxent_ne_chunker')\n","nltk.download('words')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"viVOwvUd6Yn6","executionInfo":{"status":"ok","timestamp":1668013293362,"user_tz":-60,"elapsed":623,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}},"outputId":"a1067559-d92c-451a-d40b-de0b3729a2ae"},"execution_count":94,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package maxent_ne_chunker to\n","[nltk_data] /root/nltk_data...\n","[nltk_data] Unzipping chunkers/maxent_ne_chunker.zip.\n","[nltk_data] Downloading package words to /root/nltk_data...\n","[nltk_data] Unzipping corpora/words.zip.\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":94}]},{"cell_type":"code","metadata":{"id":"LTFwmg3Dg8sO","colab":{"base_uri":"https://localhost:8080/"},"outputId":"8cead5e8-ed5b-42ee-8891-faf077fae9ed","executionInfo":{"status":"ok","timestamp":1668013317813,"user_tz":-60,"elapsed":248,"user":{"displayName":"Pranav Kasela","userId":"12416586000550437186"}}},"source":["from nltk import word_tokenize, pos_tag, ne_chunk\n","\n","input_str = \"Bill works for Apple so he went to Boston for a conference\"\n","for el in ne_chunk(pos_tag(word_tokenize(input_str))):\n"," print(el)"],"execution_count":95,"outputs":[{"output_type":"stream","name":"stdout","text":["(PERSON Bill/NNP)\n","('works', 'VBZ')\n","('for', 'IN')\n","('Apple', 'NNP')\n","('so', 'IN')\n","('he', 'PRP')\n","('went', 'VBD')\n","('to', 'TO')\n","(GPE Boston/NNP)\n","('for', 'IN')\n","('a', 'DT')\n","('conference', 'NN')\n"]}]},{"cell_type":"code","source":[],"metadata":{"id":"O-iXZ8gD6g8R"},"execution_count":null,"outputs":[]}]}