Comments (3)
I try to separate training param into 'training' and 'training_bert' in loader.py and bert.py.
now, everything seems good to me. But still not sure if I did the right thing. Can anyone give me any hints?
def get_model(token_num,
pos_num=512,
seq_len=512,
embed_dim=768,
transformer_num=12,
head_num=12,
feed_forward_dim=3072,
dropout_rate=0.1,
attention_activation=None,
feed_forward_activation=gelu,
custom_layers=None,
training=True, ### here what i change
training_bert=False, ### here what i change
lr=1e-4):
"""Get BERT model.
See: https://arxiv.org/pdf/1810.04805.pdf
:param token_num: Number of tokens.
:param pos_num: Maximum position.
:param seq_len: Maximum length of the input sequence or None.
:param embed_dim: Dimensions of embeddings.
:param transformer_num: Number of transformers.
:param head_num: Number of heads in multi-head attention in each transformer.
:param feed_forward_dim: Dimension of the feed forward layer in each transformer.
:param dropout_rate: Dropout rate.
:param attention_activation: Activation for attention layers.
:param feed_forward_activation: Activation for feed-forward layers.
:param custom_layers: A function that takes the embedding tensor and returns the tensor after feature extraction.
Arguments such as `transformer_num` and `head_num` will be ignored if `custom_layer` is not
`None`.
:param training: The built model will be returned if it is `True`, otherwise the input layers and the last feature
extraction layer will be returned.
:param lr: Learning rate.
:return: The compiled model.
"""
inputs = get_inputs(seq_len=seq_len)
embed_layer, embed_weights = get_embedding(
inputs,
token_num=token_num,
embed_dim=embed_dim,
pos_num=pos_num,
dropout_rate=dropout_rate,
trainable=training,
)
transformed = embed_layer
if custom_layers is not None:
kwargs = {}
if keras.utils.generic_utils.has_arg(custom_layers, 'trainable'):
kwargs['trainable'] = training
transformed = custom_layers(transformed, **kwargs)
else:
transformed = get_encoders(
encoder_num=transformer_num,
input_layer=transformed,
head_num=head_num,
hidden_dim=feed_forward_dim,
attention_activation=attention_activation,
feed_forward_activation=feed_forward_activation,
dropout_rate=dropout_rate,
trainable=training,
)
if not training_bert: ### here what i change
return inputs[:2], transformed
mlm_dense_layer = keras.layers.Dense(
units=embed_dim,
activation=feed_forward_activation,
name='MLM-Dense',
)(transformed)
mlm_norm_layer = LayerNormalization(name='MLM-Norm')(mlm_dense_layer)
mlm_pred_layer = EmbeddingSimilarity(name='MLM-Sim')([mlm_norm_layer, embed_weights])
masked_layer = Masked(name='MLM')([mlm_pred_layer, inputs[-1]])
extract_layer = Extract(index=0, name='Extract')(transformed)
nsp_dense_layer = keras.layers.Dense(
units=embed_dim,
activation='tanh',
name='NSP-Dense',
)(extract_layer)
nsp_pred_layer = keras.layers.Dense(
units=2,
activation='softmax',
name='NSP',
)(nsp_dense_layer)
model = keras.models.Model(inputs=inputs, outputs=[masked_layer, nsp_pred_layer])
model.compile(
optimizer=keras.optimizers.Adam(lr=lr),
loss=keras.losses.sparse_categorical_crossentropy,
)
return model
def load_trained_model_from_checkpoint(config_file,
checkpoint_file,
training=False, ### here what i change
training_bert=False, ### here what i change
seq_len=None):
"""Load trained official model from checkpoint.
:param config_file: The path to the JSON configuration file.
:param checkpoint_file: The path to the checkpoint files, should end with '.ckpt'.
:param training: If training, the whole model will be returned.
Otherwise, the MLM and NSP parts will be ignored.
:param seq_len: If it is not None and it is shorter than the value in the config file, the weights in
position embeddings will be sliced to fit the new length.
:return:
"""
with open(config_file, 'r') as reader:
config = json.loads(reader.read())
if seq_len is None:
seq_len = config['max_position_embeddings']
else:
seq_len = min(seq_len, config['max_position_embeddings'])
loader = checkpoint_loader(checkpoint_file)
model = get_model(
token_num=config['vocab_size'],
pos_num=seq_len,
seq_len=seq_len,
embed_dim=config['hidden_size'],
transformer_num=config['num_hidden_layers'],
head_num=config['num_attention_heads'],
feed_forward_dim=config['intermediate_size'],
training=training,
)
if not training_bert: ### here what i change
inputs, outputs = model
model = keras.models.Model(inputs=inputs, outputs=outputs)
model.compile(
optimizer=keras.optimizers.Adam(),
loss=keras.losses.sparse_categorical_crossentropy,
)
model.get_layer(name='Embedding-Token').set_weights([
loader('bert/embeddings/word_embeddings'),
])
model.get_layer(name='Embedding-Position').set_weights([
loader('bert/embeddings/position_embeddings')[:seq_len, :],
])
model.get_layer(name='Embedding-Segment').set_weights([
loader('bert/embeddings/token_type_embeddings'),
])
model.get_layer(name='Embedding-Norm').set_weights([
loader('bert/embeddings/LayerNorm/gamma'),
loader('bert/embeddings/LayerNorm/beta'),
])
for i in range(config['num_hidden_layers']):
model.get_layer(name='Encoder-%d-MultiHeadSelfAttention' % (i + 1)).set_weights([
loader('bert/encoder/layer_%d/attention/self/query/kernel' % i),
loader('bert/encoder/layer_%d/attention/self/query/bias' % i),
loader('bert/encoder/layer_%d/attention/self/key/kernel' % i),
loader('bert/encoder/layer_%d/attention/self/key/bias' % i),
loader('bert/encoder/layer_%d/attention/self/value/kernel' % i),
loader('bert/encoder/layer_%d/attention/self/value/bias' % i),
loader('bert/encoder/layer_%d/attention/output/dense/kernel' % i),
loader('bert/encoder/layer_%d/attention/output/dense/bias' % i),
])
model.get_layer(name='Encoder-%d-MultiHeadSelfAttention-Norm' % (i + 1)).set_weights([
loader('bert/encoder/layer_%d/attention/output/LayerNorm/gamma' % i),
loader('bert/encoder/layer_%d/attention/output/LayerNorm/beta' % i),
])
model.get_layer(name='Encoder-%d-MultiHeadSelfAttention-Norm' % (i + 1)).set_weights([
loader('bert/encoder/layer_%d/attention/output/LayerNorm/gamma' % i),
loader('bert/encoder/layer_%d/attention/output/LayerNorm/beta' % i),
])
model.get_layer(name='Encoder-%d-FeedForward' % (i + 1)).set_weights([
loader('bert/encoder/layer_%d/intermediate/dense/kernel' % i),
loader('bert/encoder/layer_%d/intermediate/dense/bias' % i),
loader('bert/encoder/layer_%d/output/dense/kernel' % i),
loader('bert/encoder/layer_%d/output/dense/bias' % i),
])
model.get_layer(name='Encoder-%d-FeedForward-Norm' % (i + 1)).set_weights([
loader('bert/encoder/layer_%d/output/LayerNorm/gamma' % i),
loader('bert/encoder/layer_%d/output/LayerNorm/beta' % i),
])
if training_bert: ### here what i change
model.get_layer(name='MLM-Dense').set_weights([
loader('cls/predictions/transform/dense/kernel'),
loader('cls/predictions/transform/dense/bias'),
])
model.get_layer(name='MLM-Norm').set_weights([
loader('cls/predictions/transform/LayerNorm/gamma'),
loader('cls/predictions/transform/LayerNorm/beta'),
])
model.get_layer(name='MLM-Sim').set_weights([
loader('cls/predictions/output_bias'),
])
model.get_layer(name='NSP-Dense').set_weights([
loader('bert/pooler/dense/kernel'),
loader('bert/pooler/dense/bias'),
])
model.get_layer(name='NSP').set_weights([
np.transpose(loader('cls/seq_relationship/output_weights')),
loader('cls/seq_relationship/output_bias'),
])
return model
from keras-bert.
I've added an argument trainable
:
load_trained_model_from_checkpoint(
...
...
training=False,
trainable=True,
)
from keras-bert.
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
from keras-bert.
Related Issues (20)
- keras-bert load pre model failed in keras 2.2.4, maybe it is a bug
- could you provide an albert loader ? HOT 1
- 如何实现在自己的数据上很少的预训练? HOT 1
- 如何提取长度一致的bert词向量? HOT 1
- Input sentence for extract_embeddings HOT 1
- List index out of range for example code HOT 2
- 在哪里可以找到之前的版本呢? HOT 2
- Keras Bert with character embeddings HOT 1
- The method get_model to get the bert model is right? HOT 4
- API reference list
- which version of keras-bert is compatible for tensorflow 1.13? HOT 2
- 关于显存巨量占用的疑惑 HOT 1
- Improve Tokenizer for uppercase text HOT 1
- Error in getting attention map HOT 3
- Masked LM pre-training HOT 1
- AttributeError: module 'keras' has no attribute 'applications' HOT 5
- module 'tensorflow.compat.v2' has no attribute '__internal__' HOT 1
- load_trained_model_from_checkpoint(config_path,checkpoint_file=checkpoint_path) AttributeError: 'tuple' object has no attribute 'layer'
- load_trained_model_from_checkpoint(config_path,checkpoint_file=checkpoint_path)
- 'adamwarmup' object has no attribute '_set_hyper'
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from keras-bert.