firstly for taking out the names

import pandas as pd
import re
from collections import Counter

# Read the data from the file
data_tash = pd.read_csv("path/to/data_tash.csv")

# Convert text to lowercase
data_tash['text_all'] = data_tash['text_all'].str.lower()

# Split text into words considering space, comma, and line breaks as delimiters
words_text_all_withlinebreaks = re.split("[ ,\n]+", " ".join(data_tash['text_all']))

# Remove special characters from words
pattern = r"^[:;!@#•·»«~}+*{?§%&>()-|=_\"°≈÷×—'“”️]+"
words_text_all_withlinebreaks_clean = [re.sub(pattern, "", word) for word in words_text_all_withlinebreaks]

# Count the occurrences of each word
words_text_count_all_withlinebreaks = Counter(words_text_all_withlinebreaks_clean)

# Convert the Counter object to a DataFrame
words_text_count_all_withlinebreaks = pd.DataFrame.from_dict(words_text_count_all_withlinebreaks, orient='index').reset_index()
words_text_count_all_withlinebreaks.columns = ['word', 'count']

# Sort the DataFrame by word count in descending order
words_text_count_all_withlinebreaks = words_text_count_all_withlinebreaks.sort_values(by='count', ascending=False)

# Calculate the midpoint of the dataset
midpoint_withlinebreaks = len(words_text_count_all_withlinebreaks) // 2

# Split the data into two parts
data_part1_midpoint_withlinebreaks = words_text_count_all_withlinebreaks.iloc[:midpoint_withlinebreaks]
data_part2_midpoint_withlinebreaks = words_text_count_all_withlinebreaks.iloc[midpoint_withlinebreaks:]

# Save each part as a CSV file
data_part1_midpoint_withlinebreaks.to_csv("data_part1_withlinebreaks.csv", index=False)
data_part2_midpoint_withlinebreaks.to_csv("data_part2_withlinebreaks.csv", index=False)





splitting one by one tokens