Hybridize Conv2DLSTMCell

Gluon
1

Can anyone tell me how to hybridize Conv2dLSTM cell?
My code is as follows:

class Net(gluon.HybridBlock):
def __init__(self, **kwargs):
    super(Net, self).__init__(**kwargs)
    with self.name_scope():

        self.cnn0 = mx.gluon.contrib.rnn.Conv2DLSTMCell(input_shape=(n_input, rows, columns),hidden_channels=n_input, activation='relu',i2h_kernel=(3,3),i2h_pad=(1,1), h2h_kernel=(3,3))
       
def hybrid_forward(self, F, x):
    
    if k==1:
        init_state =self.cnn0.begin_state(batch_size=batch_size)
        state = init_state
        k==0
    x,state= self.cnn0(x,state)

It works perfectly well before hybridize() is executed. however, later I get this error:

Epoch 1. Loss: 0.0454747959 Test MAE: ([‘mae’, ‘mse’, ‘rmse’], [0.13146833710226358, 0.04243638356417125, 0.20268271084314857]) time:35.75 s

Traceback (most recent call last):
File “Practice2.py”, line 265, in
test=evaluate_accuracy(net, test_iter)
File “Practice2.py”, line 179, in evaluate_accuracy
preds=model(data)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 540, in call
out = self.forward(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 907, in forward
return self._call_cached_op(x, *args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 797, in _call_cached_op
self._build_cache(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 749, in _build_cache
data, out = self._get_graph(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 741, in _get_graph
out = self.hybrid_forward(symbol, *grouped_inputs, **params) # pylint: disable=no-value-for-parameter
File “Practice2.py”, line 203, in hybrid_forward
x,state= self.cnn0(x,state)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 540, in call
out = self.forward(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/rnn/rnn_cell.py”, line 295, in forward
return super(RecurrentCell, self).forward(inputs, states)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 924, in forward
return self.hybrid_forward(symbol, x, *args, **params)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/contrib/rnn/conv_rnn_cell.py”, line 457, in hybrid_forward
prefix)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/contrib/rnn/conv_rnn_cell.py”, line 167, in _conv_forward
name=prefix+‘h2h’)
File “”, line 133, in Convolution
AssertionError: Argument data must be Symbol instances, but got
[[[[0…]]]]
<NDArray 3x3x92x91 @gpu(0)>

Please help me.
Thank you.

2

Please provide your full code

3

Thank you for you reply

train_iter = gluon.data.DataLoader(gluon.data.ArrayDataset(train_x, train_y), batch_size=batch_size)
valid_iter = gluon.data.DataLoader(gluon.data.ArrayDataset(valid_x, valid_y), batch_size=batch_size)
test_iter = gluon.data.DataLoader(gluon.data.ArrayDataset(test_x, test_y), batch_size=batch_size)

def evaluate_accuracy(model, dataloader):
eval_metrics_1 = mx.metric.MAE()
eval_metrics_2 = mx.metric.MSE()
eval_metrics_3 = mx.metric.RMSE()
eval_metrics = mx.metric.CompositeEvalMetric()
for child_metric in [eval_metrics_1, eval_metrics_2, eval_metrics_3]:
eval_metrics.add(child_metric)

for i, (data, label) in enumerate(dataloader):
    data = data.as_in_context(ctx)
    label = label.as_in_context(ctx)
    preds=model(data)
    #print(label.shape)
    eval_metrics.update(labels=label, preds=preds)
return eval_metrics.get()

class Net(gluon.HybridBlock):
def init(self, **kwargs):
super(Net, self).init(**kwargs)
with self.name_scope():
# layers created in name_scope will inherit name space
# from parent layer.
self.cnn0 = mx.gluon.contrib.rnn.Conv2DLSTMCell(input_shape=(n_input, rows, columns),hidden_channels=n_input, activation=‘relu’,i2h_kernel=(3,3),i2h_pad=(1,1), h2h_kernel=(3,3))
#self.cnn1 = mx.gluon.contrib.rnn.Conv2DLSTMCell(input_shape=(n_input, rows, columns),hidden_channels=n_input, activation=‘relu’,i2h_kernel=(5,5),i2h_pad=(2,2), h2h_kernel=(5,5))
self.flat1=(mx.gluon.nn.Flatten())
self.rnn1 = mx.gluon.rnn.LSTM(rnn_size, n_layer, ‘NTC’,bidirectional=True)
self.dropout=mx.gluon.nn.Dropout(dropout)
self.dense1 = mx.gluon.nn.Dense(rows*columns)

def hybrid_forward(self, F, x):
    
    if k==1:
        init_state =self.cnn0.begin_state(batch_size=batch_size)
        state = init_state
        k==0
    x,state= self.cnn0(x,state)

    
    x=self.flat1(x)
    x=x.reshape(batch_size,n_length,-1)


    x= self.rnn1(x)
    x=self.dropout(x)
    
    x=self.dense1(x)
    
    x= x.reshape(batch_size, rows,columns)        
    
    return x

net=Net()
t_loss,v_loss, net=fit(net)
net.hybridize()

test=evaluate_accuracy(net, test_iter)
print("Test : ", test)

I get error when the model is evaluated for test data.
Above part of the code is just data creation and preprocessing.
Please help.

4

It looks like you are not initializing your model. Add net.initialize(ctx = ctx) before net.hybridize()

5

Thank you for you time n kind reply…

I have initialized it… That is not mentioned here… Error is after training and hybridization on so error can’t be from initialization or anything else

It’s occurring at cnn2dlstm layer,… maybe there is some other way to hybridize cnn2dlstm layer or… Something else

Please help.

6

I’d like to know is k in your hybrid_forward

7

The exception you get here related to the fact that when you call begin_state, the result is returned as list, but in hybridized state cnn0 expects Symbol.

To fix that, traditional way is to create initial state of the RNN outside of hybrid_forward and pass it as an extra parameter.

Unfortunately, you didn’t provide values for your parameters: n_input, rows, columns, rnn_size, batch_size, n_length and fit function, so I cannot paste it in your code, but it will look something like that:

  1. Your hybrid_forward starts to accept an extra argument state, pass it to the RNN layer, and return state for the next iteration.
    def hybrid_forward(self, F, x, state):
        x, state = self.cnn0(x, state)
        ....
        return x, state
  1. In your forward pass, before calling model(x) you would receive begin_state from conv network and provide it to the call:
    for i, (data, label) in enumerate(dataloader):
        data = data.as_in_context(ctx)
        label = label.as_in_context(ctx)
        init_state = model.cnn0.begin_state(batch_size=batch_size)

        with autograd.record():
            preds, state = model(data, init_state)
  1. You need to collect state from your return and either pass it into the model during the next iteration, or create initial state again, if you know that state shouldn’t be reused.

And just a side note: since there multiple things involved with handing initial state, it is easier to complete avoid using it. I don’t know your case, but if there is no connection between different batches of your data, then the easiest way is just let MXNet to deal with the state internally on its own (it will pass zero state by default).

2 Likes
8

Thank you so much @Sergey **
** However, when I changed my code as you explained, I get a different error. Please let me know where am I going wrong I will paste the error.

error After editing:

Traceback (most recent call last):
File “Practice2.py”, line 233, in
t_loss,v_loss, net,state=fit(net)
File “Practice2.py”, line 206, in fit
Y_pred,state= model(data,state)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 540, in call
out = self.forward(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 917, in forward
return self.hybrid_forward(ndarray, x, *args, **params)
File “Practice2.py”, line 184, in hybrid_forward
x,state= self.cnn0(x,state)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 540, in call
out = self.forward(*args)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/rnn/rnn_cell.py”, line 295, in forward
return super(RecurrentCell, self).forward(inputs, states)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/block.py”, line 917, in forward
return self.hybrid_forward(ndarray, x, *args, **params)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/contrib/rnn/conv_rnn_cell.py”, line 457, in hybrid_forward
prefix)
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/gluon/contrib/rnn/conv_rnn_cell.py”, line 167, in _conv_forward
name=prefix+‘h2h’)
File “”, line 167, in Convolution
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/_ctypes/ndarray.py”, line 92, in _imperative_invoke
ctypes.byref(out_stypes)))
File “/home/test/anaconda3/lib/python3.7/site-packages/mxnet/base.py”, line 252, in check_call
raise MXNetError(py_str(_LIB.MXGetLastError()))
mxnet.base.MXNetError: [08:36:23] src/imperative/./imperative_utils.h:70: Check failed: inputs[i]->ctx().dev_mask() == ctx.dev_mask() (2 vs. 1) Operator Convolution require all inputs live on the same context. But the first argument is on cpu(0) while the 2-th argument is on gpu(0)

Please help me.

9

Yeah, you need to provide context to your begin_state like that:

state = model.cnn0.begin_state(batch_size=batch_size, ctx=ctx)

Otherwise, begin_state creates state on CPU, but your data lives on GPU. So, when your Conv2DLSTMCell receives both arrays, it cannot do the computation, because MXNet doesn’t permit computations on NDArrays which live in different contexts (and it doesn’t silently copy arrays between contexts for better memory management).

2 Likes
10

Thank you so much it worked. You are amazing.