approachingalmost
approachingalmost copied to clipboard
abhishekkrthakur
Tensorflow embedding function error
Hi, While running the tensorflow code provided in the code, i faced this error:
import os
import gc
import joblib
import pandas as pd
import numpy as np
from sklearn import metrics, preprocessing
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras.models import Model, load_model
from tensorflow.keras import callbacks
from tensorflow.keras import backend as k
from tensorflow.keras import utils
def create_model(data, catcols):
'''
this funciton returns a compiled tf.keras model for entitiy embeddings
:param data: this is a pandas dataframe
:param catcols: list of categorical column names
:return: complied tf.keras model
'''
#init the list of inputs for embedding
inputs =[]
#init the list of outputs for embedding
outputs= []
#loop over all categorical columns
for c in catcols:
#find the number of unique values in the column
num_unique_values= int(data[c].nunique())
#simple dimension of embedding calculator
#min size is half the number of unbique values
#max size is 50. max size depends on the number of values
#categories too. 50 is quite sufficient most of the times
#but if you have millions of unique values, you might need a larger dimenion
embed_dim = int(min(np.ceil((num_unique_values)/2), 50))
#simple keras input layer with size 1
inp = layers.Input(shape = (1,))
#add embedding layer to raw input
#embedding size is alwasy 1 more than unique values in input
out = layers.Embedding(num_unique_values + 1, embed_dim, name = c)(inp)
#1-d spatial dropout is the standard for embedding layers
#it can be used in nlp tasks as well
out = layers.SpatialDropout1D(0.3)(out)
#reshape the input to the dimensions of embedding
#this becomes our output layer for current feature
out = layers.Reshape(target_shape = (embed_dim,))(out)
#add input to input list
inputs.append(inp)
#add output to output list
outputs.append(out)
#concatenate all output layers
X = layers.Concatenate()(outputs)
# add a batchnorm layer
# from here, everything is up to you
# you can try different architecture
# add numerical features here or in concatonate layer
X = layers.BatchNormalization()(X)
# a bunch of dense layers with dropout
# start with 1 or two layers only
X = layers.Dense(300,activation = 'relu')(X)
X = layers.Dropout(0.3)(X)
X = layers.BatchNormalization()(X)
#using softmax and treating it as a two class problem
# sigmoid can also be used but then we need only 1 output class
y = layers.Dense(2, activation = 'softmax')(X)
model = Model(inputs = inputs ,outputs = y)
#compile the model
# we use adam and binary cross entropy
model.compile(loss = 'binary_crossentropy', optimizer = 'adam')
return model
def run(fold):
df = pd.read_csv('../input/cat_train_folds.csv')
features = [
f for f in df.columns if f not in ("id","target","kfold")
]
#fill all Na with NONE
for col in features:
df.loc[:,col] = df[col].astype(str).fillna("NONE")
#encode all features with label encoder individually
#in a live setting all label encoders need to be saved
for feat in features:
df.loc[:,feat] = df[feat].astype(str)
lbl_enc = preprocessing.LabelEncoder()
lbl_enc = lbl_enc.fit(df[feat].values)
df.loc[:, feat] = lbl_enc.fit_transform(df[feat].astype(str).values)
#get trainign data using folds
df_train= df[df.kfold != fold].reset_index(drop = True)
df_valid = df[df.kfold ==fold].reset_index(drop = True)
model = create_model(df, features)
#our features are a list of list
Xtrain = [df_train[features].values[:,k] for k in range(len(features))]
Xvalid = [df_valid[features].values[:,k] for k in range(len(features))]
ytrain = df_train.target.values
yvalid = df_train.target.values
#concert target columns to categories
#this is just binarization
ytrain_cat = utils.to_categorical(ytrain)
yvalid_cat = utils.to_categorical(yvalid)
#fit the model
model.fit(Xtrain,ytrain_cat, validation_data = (Xvalid, yvalid_cat), verbose = 1, batch_size =1024, epochs = 3)
valid_preds = model.predict(Xvalid)[:,1]
print(metrics.roc_auc_score(yvalid, valid_preds))
#clear session to free gpu memory
k.clear_session()
if __name__ == "__main__":
run(0)
run(1)
run(2)
run(3)
run(4)
The error:
(ml) sahand@sahand-System-Product-Name:~/ApproachingML/cat-in-the-dat/src$ python neural_embedding.py
2020-09-30 00:50:35.370138: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2020-09-30 00:50:42.254838: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
2020-09-30 00:50:42.275733: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2020-09-30 00:50:42.276275: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: TITAN RTX computeCapability: 7.5
coreClock: 1.77GHz coreCount: 72 deviceMemorySize: 23.65GiB deviceMemoryBandwidth: 625.94GiB/s
2020-09-30 00:50:42.276297: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2020-09-30 00:50:42.277275: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
2020-09-30 00:50:42.278136: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
2020-09-30 00:50:42.278320: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
2020-09-30 00:50:42.279236: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
2020-09-30 00:50:42.279669: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
2020-09-30 00:50:42.279787: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcudnn.so.7'; dlerror: libcudnn.so.7: cannot open shared object file: No such file or directory
2020-09-30 00:50:42.279796: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1753] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2020-09-30 00:50:42.279962: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2020-09-30 00:50:42.283430: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 3699850000 Hz
2020-09-30 00:50:42.283675: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5629cad878a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-09-30 00:50:42.283685: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
2020-09-30 00:50:42.284397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
2020-09-30 00:50:42.284406: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263]
Epoch 1/3
465/469 [============================>.] - ETA: 0s - loss: 0.4713Traceback (most recent call last):
File "neural_embedding.py", line 135, in
At this point, i think this error is due to tensorflow removing the support from list inputs. I tried to fix it by giving it a list of numpy arrays as Xtrain but i got a new error.
Xtrain = []
for k in range(len(features)):
Xtrain.append([])
Xvalid = []
for k in range(len(features)):
Xvalid.append([])
for k in range(len(features)):
Xtrain[k].append(np.asarray(df_train[features].values[:,k]))
error:
y_traincat shape 480000 y_traincat shape (480000, 2) Epoch 1/3 467/469 [============================>.] - ETA: 0s - loss: 0.4757 IndexError Traceback (most recent call last) in ----> 1 run(1)
in run(fold) 151 print(f'y_traincat shape {ytrain_cat.shape}') 152 --> 153 model.fit(Xtrain,ytrain_cat, validation_data = (Xvalid, yvalid_cat), verbose = 1, batch_size =1024, epochs = 3) 154 155
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs) 64 from tensorflow.python.ops import math_ops 65 from tensorflow.python.ops import sparse_ops ---> 66 from tensorflow.python.ops import summary_ops_v2 67 from tensorflow.python.ops import variables 68 from tensorflow.python.ops.ragged import ragged_concat_ops
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
860 A more detailed description of unpacking behavior for iterator types
861 (Dataset, generator, Sequence) is given below.
--> 862 y: Target data. Like the input data x,
863 it could be either Numpy array(s) or TensorFlow tensor(s).
864 It should be consistent with x (you cannot have Numpy inputs and
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs) 64 from tensorflow.python.ops import math_ops 65 from tensorflow.python.ops import sparse_ops ---> 66 from tensorflow.python.ops import summary_ops_v2 67 from tensorflow.python.ops import variables 68 from tensorflow.python.ops.ragged import ragged_concat_ops
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in evaluate(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict)
1044 data_adapter.unpack_x_y_sample_weight(validation_data))
1045
-> 1046 with self.distribute_strategy.scope(),
1047 training_utils.RespectCompiledTrainableState(self):
1048 # Creates a tf.data.Dataset and handles batch and epoch iteration.
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in init(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model) 1097 if steps_per_execution is None: 1098 self._steps_per_execution = 1 -> 1099 self._steps_per_execution_value = 1 1100 else: 1101 self._steps_per_execution = steps_per_execution
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in select_data_adapter(x, y) 955 956 ALL_ADAPTER_CLS = [ --> 957 ListsOfScalarsDataAdapter, TensorLikeDataAdapter, 958 GenericArrayLikeDataAdapter, DatasetAdapter, 959 GeneratorDataAdapter, KerasSequenceAdapter, CompositeTensorDataAdapter,
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in (.0) 955 956 ALL_ADAPTER_CLS = [ --> 957 ListsOfScalarsDataAdapter, TensorLikeDataAdapter, 958 GenericArrayLikeDataAdapter, DatasetAdapter, 959 GeneratorDataAdapter, KerasSequenceAdapter, CompositeTensorDataAdapter,
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in can_handle(x, y) 617 @staticmethod 618 def can_handle(x, y=None): --> 619 handles_x = ListsOfScalarsDataAdapter._is_list_of_scalars(x) 620 handles_y = True 621 if y is not None:
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in _is_list_of_scalars(inp) 628 return True 629 if isinstance(inp, (list, tuple)): --> 630 return ListsOfScalarsDataAdapter._is_list_of_scalars(inp[0]) 631 return False 632
~/anaconda3/envs/ml/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py in _is_list_of_scalars(inp) 628 return True 629 if isinstance(inp, (list, tuple)): --> 630 return ListsOfScalarsDataAdapter._is_list_of_scalars(inp[0]) 631 return False 632
IndexError: list index out of range
I get a Value Error on running the above code
ValueError: A Concatenate layer should be called on a list of at least 2 inputs
ValueError Traceback (most recent call last)
~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in call(self, *args, **kwargs)
895 # Build layer if applicable (if the build method has been
896 # overridden).
--> 897 self._maybe_build(inputs)
898 cast_inputs = self._maybe_cast_inputs(inputs)
899
~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs) 2414 # operations. 2415 with tf_utils.maybe_init_scope(self): -> 2416 self.build(input_shapes) # pylint:disable=not-callable 2417 # We must set also ensure that the layer is marked as built, and the build 2418 # shape is stored since user defined build functions may not be calling
~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py in wrapper(instance, input_shape)
314 if input_shape is not None:
315 input_shape = convert_shapes(input_shape, to_tuples=True)
--> 316 output_shape = fn(instance, input_shape)
317 # Return shapes from fn as TensorShapes.
318 if output_shape is not None:
~/miniconda3/envs/ml/lib/python3.7/site-packages/tensorflow/python/keras/layers/merge.py in build(self, input_shape)
491 # Used purely for shape validation.
492 if not isinstance(input_shape[0], tuple) or len(input_shape) < 2:
--> 493 raise ValueError('A Concatenate layer should be called '
494 'on a list of at least 2 inputs')
495 if all(shape is None for shape in input_shape):
