swadhinbiswas datasync · 2/3/2024
6e1b220
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from collections import Counter
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
# Define a class to find plagiarism
class FindPlager():
def __init__(self,text1,text2):
self.text1 = text1
self.text2 = text2
def preprocess(self,text):
# Tokenize the text
tokens = word_tokenize(text.lower())
# Remove stop words and non-alphanumeric characters
stop_words = set(stopwords.words('english'))
tokens = [token for token in tokens if token.isalnum() and token not in stop_words]
# Lemmatize tokens
lemmatizer = WordNetLemmatizer()
tokens = [lemmatizer.lemmatize(token) for token in tokens]
return tokens
def get_similarity_score(self,text1, text2):
tokens1 = self.preprocess(text1)
tokens2 = self.preprocess(text2)
# Calculate Jaccard similarity
intersection = set(tokens1).intersection(set(tokens2))
union = set(tokens1).union(set(tokens2))
jaccard_similarity = len(intersection) / len(union)
return jaccard_similarity
def check_plagiarism(self,threshold=0.7):
similarity_score = self.get_similarity_score(self.text1, self.text2)
if similarity_score >= threshold:
return True, similarity_score
else:
return False, similarity_score
def main(self):
return self.check_plagiarism()