Comments (2)
贴代码
package org.apache.solr.request;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.WeakHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BigReUsedBuffer {
private static Logger log = LoggerFactory.getLogger(BigReUsedBuffer.class);
private final static int intervalBits = 18;
private final static int intervalMask = 0xffffffff >>> (32 - intervalBits);
public final static int interval = 1 << intervalBits;// 1024*1024
private Integer indexFree = 0;
private Integer indexUsed = 0;
private Integer mallocTimes = 0;
private WeakHashMap<Integer, BlockInterface> userd = new WeakHashMap<Integer, BlockInterface>();
private HashMap<Integer, BlockInterface> free = new HashMap<Integer, BlockInterface>();
private Object lock = new Object();
public void free(BlockArray<T> data) {
synchronized (lock) {
for (Integer i : data.indexlist) {
BlockInterface<T> d = userd.remove(i);
if (d != null) {
indexFree++;
if (indexFree > 100000000) {
indexFree = 0;
}
this.free.put(indexFree, d);
}
}
}
}
public void free(BlockData<T> data) {
synchronized (lock) {
BlockInterface<T> d = userd.remove(data.index);
if (d != null) {
indexFree++;
if (indexFree > 100000000) {
indexFree = 0;
}
this.free.put(indexFree, d);
}
}
}
public BlockData<T> callocBlock(CreateArr<T> c, T init)
{
synchronized (lock) {
this.userd.get(0);
ArrayList<Integer> rmlist = new ArrayList<Integer>();
BlockInterface<T> data = null;
for (Entry<Integer, BlockInterface<T>> e : this.free.entrySet()) {
rmlist.add(e.getKey());
data = e.getValue();
break;
}
if (data==null) {
data = c.create(interval);
mallocTimes++;
}
for (Integer i : rmlist) {
this.free.remove(i);
}
indexUsed++;
if (indexUsed > 100000000) {
indexUsed = 0;
}
userd.put(indexUsed, data);
for (int i = 0; i < interval; i++) {
data.set(i, init);
}
log.info("BigByteBuffer callocBlock used:"+this.userd.size()+",free:"+free.size()+",mallocTimes:"+mallocTimes);
return new BlockData<T>(indexUsed, data);
}
}
public BlockArray<T> calloc(int size, CreateArr<T> c, T init) {
synchronized (lock) {
this.userd.get(0);
ArrayList<Integer> rmlist = new ArrayList<Integer>();
int block = size / interval + 1;
Integer[] indexlist = new Integer[block];
BlockInterface<T>[] data = c.createBlocks(block);
int index = 0;
for (Entry<Integer, BlockInterface<T>> e : this.free.entrySet()) {
if (index >= block) {
break;
}
rmlist.add(e.getKey());
data[index] = e.getValue();
index++;
}
for (int i = index; i < block; i++) {
data[i] = c.create(interval);
mallocTimes++;
}
for (Integer i : rmlist) {
this.free.remove(i);
}
for (int i = 0; i < block; i++) {
indexUsed++;
if (indexUsed > 100000000) {
indexUsed = 0;
}
userd.put(indexUsed, data[i]);
indexlist[i] = indexUsed;
}
for (int i = 0; i < data.length; i++) {
for (int j = 0; j < interval; j++) {
data[i].set(j, init) ;
}
}
log.info("BigByteBuffer calloc used:"+this.userd.size()+",free:"+free.size()+",mallocTimes:"+mallocTimes);
return new BlockArray<T>(indexlist, data, size);
}
}
public static CreateArr<Byte> BYTE_CREATE=new CreateArr<Byte>() {
@Override
public BlockInterface<Byte> create(int size) {
return new BlockByte(size);
}
@Override
public BlockInterface<Byte>[] createBlocks(int size) {
return new BlockByte[size];
}
};
public static CreateArr<Integer> INT_CREATE=new CreateArr<Integer>() {
@Override
public BlockInterface<Integer> create(int size) {
return new BlockInteger(size);
}
@Override
public BlockInterface<Integer>[] createBlocks(int size) {
return new BlockInteger[size];
}
};
public static CreateArr<Long> LONG_CREATE=new CreateArr<Long>() {
@Override
public BlockInterface<Long> create(int size) {
return new BlockLong(size);
}
@Override
public BlockInterface<Long>[] createBlocks(int size) {
return new BlockLong[size];
}
};
public static CreateArr<Double> DOUBLE_CREATE=new CreateArr<Double>() {
@Override
public BlockInterface<Double> create(int size) {
return new BlockDouble(size);
}
@Override
public BlockInterface<Double>[] createBlocks(int size) {
return new BlockDouble[size];
}
};
public static interface CreateArr<T> {
public BlockInterface<T> create(int size);
public BlockInterface<T>[] createBlocks(int size);
}
public static class BlockData<K> {
public Integer index;
public BlockInterface<K> data;
public BlockData(Integer indexlist, BlockInterface<K> data) {
this.index = indexlist;
this.data = data;
}
}
public static class BlockInteger implements BlockInterface<Integer>
{
int[] data;
public BlockInteger(int size)
{
data=new int[size];
}
@Override
public Integer get(int i) {
return data[i];
}
@Override
public void set(int i, Integer v) {
data[i]=v;
}
}
public static class BlockLong implements BlockInterface<Long>
{
long[] data;
public BlockLong(int size)
{
data=new long[size];
}
@Override
public Long get(int i) {
return data[i];
}
@Override
public void set(int i, Long v) {
data[i]=v;
}
}
public static class BlockDouble implements BlockInterface<Double>
{
double[] data;
public BlockDouble(int size)
{
data=new double[size];
}
@Override
public Double get(int i) {
return data[i];
}
@Override
public void set(int i, Double v) {
data[i]=v;
}
}
public static class BlockByte implements BlockInterface<Byte>
{
public byte[] data;
public BlockByte(int size)
{
data=new byte[size];
}
@Override
public Byte get(int i) {
return data[i];
}
@Override
public void set(int i, Byte v) {
data[i]=v;
}
}
public static interface BlockInterface<K>{
public K get(int i);
public void set(int i,K v);
}
public static class BlockArray<K> {
private int size = 0;
private Integer[] indexlist;
private BlockInterface<K>[] data;
public BlockArray(Integer[] indexlist, BlockInterface<K>[] data, int size) {
this.indexlist = indexlist;
this.data = data;
this.size = size;
}
public K get(int i) {
int block = i >>> intervalBits;
int pos = i & intervalMask;
return data[block].get(pos);
}
public void set(int i, K v) {
int block = i >>> intervalBits;
int pos = i & intervalMask;
data[block].set(pos, v);
}
public int getSize() {
return size;
}
}
}
from higo.
已经上线
from higo.
Related Issues (20)
- 海狗的solr心跳使用set_ephemeral_node
- 分区除了default,day,month外支持single分区 HOT 1
- 海狗的shards支持replication
- 建表语句使用Sql,之前通过编辑schema.xml的方式太麻烦了 HOT 1
- cacheField根据目录打散,利用多磁盘
- adhoc二期 每个表起始时间的接口
- 监控添加进程ID
- 重构-监控每个分区,以及起始几天的记录数这段代码独立出来
- maven jar包不在完全合并成独立的 HOT 1
- thedate 默认从文件路径中读取 HOT 1
- skipTo使用的是terms,这里面用到了大量的clone HOT 1
- sql的order by解析好像有问题,没起作用 HOT 2
- 一次跨越很多个分区的时候-查询报错的解决
- 超过1万组则为近似的排序和计算,有可能存在计算的结果不正确的情况 HOT 4
- 多列groupby 目前只能按照统计字段排序,和group整体排序
- 超过1万组则为近似的排序和计算,排序排序考虑周边的200个
- 排序整理重构 HOT 2
- 对外SQL的接口类名更改为HigoService
- 云梯升级后 我没有重新load配置文件 我记录下此BUG
- 多列group by的排序要按照不同的数据类型进行排序
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from higo.