Hey @Santosh-Gupta you can use following piece of function to get the sentence embedding. It's the average of token level embedding.

def get_bio_bert_embedding(sentence):
  
  outF = open("/input.txt", "w")
  outF.write(sentence)
  outF.close()
  
  os.system('python3 extract_features.py \
   --input_file=/input.txt \
   --vocab_file=/content/biobert/biobert_pubmed/vocab.txt \
   --bert_config_file=/content/biobert/biobert_pubmed/bert_config.json \
   --init_checkpoint=/content/biobert/biobert_pubmed/biobert_model.ckpt \
   --output_file=/output.jsonl')

  with open('/output.jsonl') as f:
    d = json.load(f)
    
  sentence_vector = np.zeros(768).tolist()
  for i in range (1, len(d['features'])-1):
    sentence_vector = sentence_vector + d['features'][i]['layers'][0]['values']
  
  number_of_tokens = len(d['features']) - 2
  for elem in sentence_vector:
    elem = elem/number_of_tokens
  return sentence_vector

Thanks for the code. I slightly modified the script and wrote the following code. Assuming that we have a collection of documents stored in a text file such that each document is stored in one line, the following code gives you a list of embeddings (embedding_vectors) of these documents:

import os
import json
import pickle
import numpy as np

# extract_features.py is a BERT class
os.system('python3 ' + ' extract_features.py \
            --input_file=input.txt \
            --vocab_file=vocab.txt \
            --bert_config_file=bert_config.json \
            --init_checkpoint=model.ckpt-1000000 \
            --output_file=output.json')

embedding_vectors = []
print("[progress] Processed records: ")
with open('output.json') as f:
    # each line is a json which has embedding information of a single document/description
    for line in f:
        d = json.loads(line)
        # 768 is hidden size
        doc_vector = np.zeros(768)
        
        # starting from 1 to skip [CLS] and to -1 to skip [SEP]
        for i in range (1, len(d['features'])-1):
            # feature vector of the current token
            feature_vector = np.array(d['features'][i]['layers'][0]['values'])
        
            # adding to the document vector
            doc_vector = doc_vector + feature_vector
    
        # -2 is for excluding [CLS] and [SEP] tokens
        number_of_tokens = len(d['features']) - 2
    
        # since we want to compute the average of vector representations of tokens
        doc_vector = np.divide(doc_vector, number_of_tokens)
    
        embedding_vectors.append(doc_vector)

print("[log] Saving the embedding vectors into file...")
# saving the embedding vectors in a pickle file
with open("embedding_vector.pkl", "wb") as f:
    pickle.dump(embedding_vectors, f)

n = len(embedding_vectors)
dim = len(embedding_vectors[0])
print("Number of vectors: " + str(n))
print("Vector dimension: " + str(dim))

Retreiving Embedding vector #23

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions