Giter VIP home page Giter VIP logo

Comments (3)

yhmo avatar yhmo commented on August 16, 2024

Test script:

import random
import time

import numpy as np

from pymilvus import (
    connections,
    FieldSchema, CollectionSchema, DataType,
    Collection,
    utility,
)


HOST = 'localhost'
PORT = '19530'

connections.connect(host=HOST, port=PORT)

F16_COLLECTION = "f16_col"

DIM = 768
METRIC_TYPE = "L2"

ID_FIELD = "id"
VECTOR_FIELD = "vector"

def gen_fp16_vectors(num):
    raw_vectors = []
    fp16_vectors = []
    for _ in range(num):
        raw_vector = [random.random() for _ in range(DIM)]
        raw_vectors.append(raw_vector)
        fp16_vector = np.array(raw_vector, dtype=np.float16)
        fp16_vectors.append(fp16_vector)
    return raw_vectors, fp16_vectors

def create_collection():
    if utility.has_collection(F16_COLLECTION):
        utility.drop_collection(F16_COLLECTION)

    fields = [
        FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
        FieldSchema(name="vector", dtype=DataType.FLOAT16_VECTOR, dim=DIM),
    ]
    schema = CollectionSchema(fields=fields)
    collection = Collection(name=F16_COLLECTION, schema=schema)
    print(f"Collection '{F16_COLLECTION}' created")


def prepare_data():
    collection_16 = Collection(name=F16_COLLECTION)
    count = 10000
    for i in range(100):
        raw_vectors, fp16_vectors = gen_fp16_vectors(count)
        collection_16.insert(data=[
            fp16_vectors,
        ])
        print(f"insert batch {i}")
    print("insert done")

    time.sleep(5)
    collection_16.flush()
    print("flush done")

    start = time.time()
    index_params = {
        'metric_type': METRIC_TYPE,
        'index_type': "HNSW",
        'params': {"M": 8, "efConstruction": 200},
    }
    collection_16.create_index(field_name=VECTOR_FIELD, index_params=index_params)
    utility.wait_for_index_building_complete(collection_name=F16_COLLECTION)
    print("index done")

    end = time.time()
    print(f"fp16 index time cost: {end-start} seconds")


if __name__ == '__main__':
    create_collection()
    prepare_data()

from milvus.

yanliang567 avatar yanliang567 commented on August 16, 2024

/assign @cqy123456
/unassign

from milvus.

cqy123456 avatar cqy123456 commented on August 16, 2024

/assign @yhmo
use latest 2.4 to try, fp16 and bf16 add simd support in latest 2.4.

from milvus.

Related Issues (20)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.