token_perplexity = (probs.log() * -1.0).exp()
 print(f"Token Perplexity: {token_perplexity}")

sentence_perplexity = (probs.log() * -1.0).mean().exp().item()
 print(f"Sentence Perplexity: {sentence_perplexity:.2f}")

def get_pointwise_loss(self, inputs: List[str], tok):
    self.model.eval()
    all_probs = []
    with torch.inference_mode():
        for input in inputs:
            ids_list: List[int] = tok.encode(input).ids
            # ids has shape (1, len(ids_list))
            ids: Torch.Tensor = torch.tensor(ids_list, device=self.device).unsqueeze(0)
 
            # probs below is the probability that the token at that location
            # completes the sentence (in ids) so far.
            y = self.model(ids)
            criterion = nn.CrossEntropyLoss(reductinotallow='none', ignore_index=0)
 
            # Compute the loss starting from the 2nd token in the model's output.
            loss = criterion(y[:,:,:-1], ids[:,1:])
 
            # To compute the probability of each token, we need to compute the
            # negative of the log loss and exponentiate it.
            loss = loss * -1.0
 
            # Set the probabilities that we are not interested in to -inf.
            # This is done to make the softmax set these values to 0.0
            loss[loss == 0.0] = float("-inf")
 
            # probs holds the probability of each token's prediction
            # starting from the 2nd token since we don't want to include
            # the probability of the model predicting the beginning of
            # a sentence given no existing sentence context.
            #
            # To compute perplexity, we should probably ignore the first
            # handful of predictions of the model since there's insufficient
            # context. We don’t do that here, though.
            probs = loss.exp()
            all_probs.append(probs)
        #
    #
    return all_probs
 #

def get_html_for_token_perplexity(tok, sentence, tok_ppx, model_ppx):
    tokens = tok.encode(sentence).tokens
    ids = tok.encode(sentence).ids
    cleaned_tokens = []
    for word in tokens:
        m = list(map(ord, word))
        m = list(map(lambda x: x if x != 288 else ord(' '), m))
        m = list(map(chr, m))
        m = ''.join(m)
        cleaned_tokens.append(m)
    #
    html = [
        f"<span>{cleaned_tokens[0]}</span>",
    ]
    for ct, ppx in zip(cleaned_tokens[1:], tok_ppx):
        color = "black"
        if ppx.item() >= 0:
            if ppx.item() <= model_ppx * 1.1:
                color = "red"
            else:
                color = "green"
            #
        #
        html.append(f"<span style='color:{color};'>{ct}</span>")
    #
    return "".join(html)
 #

import nltk
 from collections import Counter
 from sklearn.feature_extraction.text import CountVectorizer
 from textblob import TextBlob
 import numpy as np
 
 text = "This is some sample text to detect. This is a text written by a human? or is it?!"
 
 # 1. N-gram Analysis
 def ngram_analysis(text, n=2):
    n_grams = nltk.ngrams(text.split(), n)
    freq_dist = nltk.FreqDist(n_grams)
    print(freq_dist)
 
 ngram_analysis(text)
 
 # 2. Perplexity
 # Perplexity calculation typically requires a language model
 # This is just an example of how perplexity might be calculated given a probability distribution
 def perplexity(text, model=None):
    # This is a placeholder. In practice, we would use the model to get the probability of each word
    prob_dist = nltk.FreqDist(text.split())
    entropy = -1 * sum([p * np.log(p) for p in prob_dist.values()])
    return np.exp(entropy)
 
 print(perplexity(text))
 
 # 3. Burstiness
 def burstiness(text):
    word_counts = Counter(text.split())
    burstiness = len(word_counts) / np.std(list(word_counts.values()))
    return burstiness
 
 print(burstiness(text))
 
 # 4. Stylometry
 def stylometry(text):
    blob = TextBlob(text)
    avg_sentence_length = sum(len(sentence.words) for sentence in blob.sentences) / len(blob.sentences)
    passive_voice = text.lower().count('was') + text.lower().count('were')
    vocabulary_richness = len(set(text.split())) / len(text.split())
    return avg_sentence_length, passive_voice, vocabulary_richness
 
 print(stylometry(text))
 
 # 5. Consistency and Coherence Analysis
 # Again, this is a simple way to explain the idea of calculating the consistency of a text. In reality, more complex algorithms are used.
 def consistency(text):
    sentences = text.split(".")
    topics = [sentence.split()[0] for sentence in sentences if sentence]
    topic_changes = len(set(topics))
    return topic_changes
 
 print(consistency(text))